Pass-through/transparent management interfaces (#268)

* vrnetlab: Add pass-through management interfaces

* vjunos: Add pass-through management interface support

* vrnetlab: Use JSON output of iproute2

* vrnetlab: Add exception for serial console ports 5000-5007 for transparent mode mgmt interface

* vrnetlab: Remove non-working port 5000 tc mirred exception, redirect to correct interface

* vrnetlab: Use tc clsact qdisc and flower matching as best practice

* vrnetlab: Re-add workaround for serial ports in transparent mgmt mode

* vrnetlab: Add IPv6 support to management address/gw functions

* vjunos: Add IPv6 management addresses, fix v4 address templating

* vrnetlab: Set dummy IPv6 address/gw for hostfwd management

README.md

@@ -39,6 +39,22 @@ Other connection mode values are:
 * ovs-bridge - same as a regular bridge, but uses OvS. Can pass LACP traffic.
 * macvtap
 
+## Management interface
+
+There are two types of management connectivity for NOS VMs: _pass-through_ and _host-forwarded_ (legacy) management interfaces.
+
+_Pass-through management_ interfaces allows the use of the assigned management IP within the NOS VM, management traffic is transparently passed through to the VM, and the NOS configuration can accurately reflect the management IP. However, it is no longer possible to send or receive traffic directly in the vrnetlab container (e.g. for installing additional packages within the container), other than to pre-defined exceptions, such as the QEMU serial port on TCP port 5000.
+
+NOSes defaulting to _pass-through_ management interfaces are:
+- All vJunos routers
+
+In case of _host-forwarded_ management interfaces, certain ports are forwarded to the NOS VM IP, which is always 10.0.0.15/24. The management gateway in this case is 10.0.0.2/24, and outgoing traffic is NATed to the container management IP. This management interface connection mode does not allow for traffic such as LLDP to pass through the management interface.
+
+NOSes defaulting to _host-forwarded_ management interfaces are:
+- Every NOS not listed as pass-through management
+
+It is possible to change from the default management interface mode by setting the `CLAB_MGMT_PASSTHROUGH` environment variable to 'true' or 'false', however, it is left up to the user to provide a startup configuration compatible with the requested mode.
+
 ## Which vrnetlab routers are supported?
 Since the changes we made in this fork are VM specific, we added a few popular routing products:
 

common/vrnetlab.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import datetime
+import ipaddress
 import json
 import logging
 import math
@@ -10,7 +11,6 @@
 import subprocess
 import telnetlib
 import time
-import ipaddress
 from pathlib import Path
 
 MAX_RETRIES = 60
@@ -78,6 +78,7 @@ def __init__(
         provision_pci_bus=True,
         cpu="host",
         smp="1",
+        mgmt_passthrough=False,
         min_dp_nics=0,
     ):
         self.logger = logging.getLogger()
@@ -109,16 +110,32 @@ def __init__(
         # to have them allocated sequential from eth1
         self.highest_provisioned_nic_num = 0
 
+        # Whether the management interface is pass-through or host-forwarded
+        self.mgmt_nic_passthrough = mgmt_passthrough
+        mgmt_passthrough_override = os.environ.get("CLAB_MGMT_PASSTHROUGH", "")
+        if mgmt_passthrough_override:
+            self.mgmt_nic_passthrough = mgmt_passthrough_override.lower() == "true"
+
+        # Populate management IP and gateway
+        if self.mgmt_nic_passthrough:
+            self.mgmt_address_ipv4, self.mgmt_address_ipv6 = self.get_mgmt_address()
+            self.mgmt_gw_ipv4, self.mgmt_gw_ipv6 = self.get_mgmt_gw()
+        else:
+            self.mgmt_address_ipv4 = "10.0.0.15/24"
+            self.mgmt_address_ipv6 = "2001:db8::2/64"
+            self.mgmt_gw_ipv4 = "10.0.0.2"
+            self.mgmt_gw_ipv6 = "2001:db8::1"
+
         self.insuffucient_nics = False
         self.min_nics = 0
         # if an image needs minimum amount of dataplane nics to bootup, specify
         if min_dp_nics:
             self.min_nics = min_dp_nics
 
         # management subnet properties, defaults
-        self.mgmt_subnet    = "10.0.0.0/24"
-        self.mgmt_host_ip   = 2
-        self.mgmt_guest_ip  = 15
+        self.mgmt_subnet = "10.0.0.0/24"
+        self.mgmt_host_ip = 2
+        self.mgmt_guest_ip = 15
 
         #  Default TCP ports forwarded (TODO tune per platform):
         #  80    - http
@@ -129,7 +146,7 @@ def __init__(
         #  9339  - iana gnmi/gnoi
         #  32767 - gnmi/gnoi juniper
         #  57400 - nokia gnmi/gnoi
-        self.mgmt_tcp_ports = [80,443,830,6030,8080,9339,32767,57400]
+        self.mgmt_tcp_ports = [80, 443, 830, 6030, 8080, 9339, 32767, 57400]
 
         # we setup pci bus by default
         self.provision_pci_bus = provision_pci_bus
@@ -299,20 +316,51 @@ def create_tc_tap_ifup(self):
         ip link set $TAP_IF mtu 65000
 
         # create tc eth<->tap redirect rules
-        tc qdisc add dev eth$INDEX ingress
-        tc filter add dev eth$INDEX parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev tap$INDEX
+        tc qdisc add dev eth$INDEX clsact
+        tc filter add dev eth$INDEX ingress flower action mirred egress redirect dev tap$INDEX
 
-        tc qdisc add dev $TAP_IF ingress
-        tc filter add dev $TAP_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev eth$INDEX
+        tc qdisc add dev $TAP_IF clsact
+        tc filter add dev $TAP_IF ingress flower action mirred egress redirect dev eth$INDEX
         """
 
         with open("/etc/tc-tap-ifup", "w") as f:
             f.write(ifup_script)
         os.chmod("/etc/tc-tap-ifup", 0o777)
 
+    def create_tc_tap_mgmt_ifup(self):
+        """Create tap ifup script that is used in tc datapath mode, specifically for the management interface"""
+        ifup_script = """#!/bin/bash
+
+        ip link set tap0 up
+        ip link set tap0 mtu 65000
+
+        # create tc eth<->tap redirect rules
+
+        tc qdisc add dev eth0 clsact
+        # exception for TCP ports 5000-5007
+        tc filter add dev eth0 ingress prio 1 protocol ip flower ip_proto tcp dst_port 5000-5007 action pass
+        # mirror ARP traffic to container
+        tc filter add dev eth0 ingress prio 2 protocol arp flower action mirred egress mirror dev tap0
+        # redirect rest of ingress traffic of eth0 to egress of tap0
+        tc filter add dev eth0 ingress prio 3 flower action mirred egress redirect dev tap0
+
+        tc qdisc add dev tap0 clsact
+        # redirect all ingress traffic of tap0 to egress of eth0
+        tc filter add dev tap0 ingress flower action mirred egress redirect dev eth0
+
+        # clone management MAC of the VM
+        ip link set dev eth0 address {MGMT_MAC}
+        """
+
+        ifup_script = ifup_script.replace("{MGMT_MAC}", self.mgmt_mac)
+
+        with open("/etc/tc-tap-mgmt-ifup", "w") as f:
+            f.write(ifup_script)
+        os.chmod("/etc/tc-tap-mgmt-ifup", 0o777)
+
     def gen_mgmt(self):
         """Generate qemu args for the mgmt interface(s)
-        
+
         Default TCP ports forwarded:
           80    - http
           443   - https
@@ -323,34 +371,98 @@ def gen_mgmt(self):
           32767 - gnmi/gnoi juniper
           57400 - nokia gnmi/gnoi
         """
-        if self.mgmt_host_ip+1>=self.mgmt_guest_ip:
-            self.logger.error("Guest IP (%s) must be at least 2 higher than host IP(%s)", 
-                              self.mgmt_guest_ip, self.mgmt_host_ip)
+        if self.mgmt_host_ip + 1 >= self.mgmt_guest_ip:
+            self.logger.error(
+                "Guest IP (%s) must be at least 2 higher than host IP(%s)",
+                self.mgmt_guest_ip,
+                self.mgmt_host_ip,
+            )
 
         network = ipaddress.ip_network(self.mgmt_subnet)
         host = str(network[self.mgmt_host_ip])
-        dns = str(network[self.mgmt_host_ip+1])
+        dns = str(network[self.mgmt_host_ip + 1])
         guest = str(network[self.mgmt_guest_ip])
 
         res = []
-        # mgmt interface is special - we use qemu user mode network with DHCP
         res.append("-device")
-        mac = (
+        self.mgmt_mac = (
             "c0:00:01:00:ca:fe"
             if getattr(self, "_static_mgmt_mac", False)
             else gen_mac(0)
         )
-        res.append(self.nic_type + f",netdev=p00,mac={mac}")
+
+        res.append(self.nic_type + f",netdev=p00,mac={self.mgmt_mac}")
         res.append("-netdev")
-        res.append(
-            f"user,id=p00,net={self.mgmt_subnet},host={host},dns={dns},dhcpstart={guest}," +
-            f"hostfwd=tcp:0.0.0.0:22-{guest}:22," +   # ssh
-            f"hostfwd=udp:0.0.0.0:161-{guest}:161," +  # snmp
-            (",".join([ f"hostfwd=tcp:0.0.0.0:{p}-{guest}:{p}" for p in self.mgmt_tcp_ports ])) +
-            ",tftp=/tftpboot"
-        )
+
+        if self.mgmt_nic_passthrough:
+            # mgmt interface is passthrough - we just create a normal mirred tap interface
+            res.append(
+                "tap,id=p00,ifname=tap0,script=/etc/tc-tap-mgmt-ifup,downscript=no"
+            )
+            self.create_tc_tap_mgmt_ifup()
+        else:
+            # mgmt interface is host-forwarded - we use qemu user mode network
+            # with hostfwd rules to forward ports from the host to the guest
+            res.append(
+                f"user,id=p00,net={self.mgmt_subnet},host={host},dns={dns},dhcpstart={guest},"
+                + f"hostfwd=tcp:0.0.0.0:22-{guest}:22,"  # ssh
+                + f"hostfwd=udp:0.0.0.0:161-{guest}:161,"  # snmp
+                + (
+                    ",".join(
+                        [
+                            f"hostfwd=tcp:0.0.0.0:{p}-{guest}:{p}"
+                            for p in self.mgmt_tcp_ports
+                        ]
+                    )
+                )
+                + ",tftp=/tftpboot"
+            )
+
         return res
 
+    def get_mgmt_address(self):
+        """Returns the IPv4 and IPv6 address of the eth0 interface of the container"""
+        stdout, _ = run_command(["ip", "--json", "address", "show", "dev", "eth0"])
+        command_json = json.loads(stdout.decode("utf-8"))
+        intf_addrinfos = command_json[0]["addr_info"]
+
+        mgmt_cidr_v4 = None
+        mgmt_cidr_v6 = None
+        for addrinfo in intf_addrinfos:
+            if addrinfo["family"] == "inet" and addrinfo["scope"] == "global":
+                mgmt_address_v4 = addrinfo["local"]
+                mgmt_prefixlen_v4 = addrinfo["prefixlen"]
+                mgmt_cidr_v4 = mgmt_address_v4 + "/" + str(mgmt_prefixlen_v4)
+            if addrinfo["family"] == "inet6" and addrinfo["scope"] == "global":
+                mgmt_address_v6 = addrinfo["local"]
+                mgmt_prefixlen_v6 = addrinfo["prefixlen"]
+                mgmt_cidr_v6 = mgmt_address_v6 + "/" + str(mgmt_prefixlen_v6)
+
+        if not mgmt_cidr_v4:
+            raise ValueError("No IPv4 address set on management interface eth0!")
+
+        return mgmt_cidr_v4, mgmt_cidr_v6
+
+    def get_mgmt_gw(self):
+        """Returns the IPv4 and IPv6 default gateways of the container, used for generating the management default route"""
+        stdout_v4, _ = run_command(["ip", "--json", "-4", "route", "show", "default"])
+        command_json_v4 = json.loads(stdout_v4.decode("utf-8"))
+        try:
+            mgmt_gw_v4 = command_json_v4[0]["gateway"]
+        except IndexError as e:
+            raise IndexError(
+                "No default gateway route on management interface eth0!"
+            ) from e
+
+        stdout_v6, _ = run_command(["ip", "--json", "-6", "route", "show", "default"])
+        command_json_v6 = json.loads(stdout_v6.decode("utf-8"))
+        try:
+            mgmt_gw_v6 = command_json_v6[0]["gateway"]
+        except IndexError:
+            mgmt_gw_v6 = None
+
+        return mgmt_gw_v4, mgmt_gw_v6
+
     def nic_provision_delay(self) -> None:
         self.logger.debug(
             f"number of provisioned data plane interfaces is {self.num_provisioned_nics}"
@@ -526,7 +638,9 @@ def restart(self):
         self.stop()
         self.start()
 
-    def wait_write(self, cmd, wait="__defaultpattern__", con=None, clean_buffer=False, hold=""):
+    def wait_write(
+        self, cmd, wait="__defaultpattern__", con=None, clean_buffer=False, hold=""
+    ):
         """Wait for something on the serial port and then send command
 
         Defaults to using self.tn as connection but this can be overridden
@@ -548,8 +662,10 @@ def wait_write(self, cmd, wait="__defaultpattern__", con=None, clean_buffer=Fals
             self.logger.trace(f"waiting for '{wait}' on {con_name}")
             res = con.read_until(wait.encode())
 
-            while (hold and (hold in res.decode())):
-                self.logger.trace(f"Holding pattern '{hold}' detected: {res.decode()}, retrying in 10s...")
+            while hold and (hold in res.decode()):
+                self.logger.trace(
+                    f"Holding pattern '{hold}' detected: {res.decode()}, retrying in 10s..."
+                )
                 con.write("\r".encode())
                 time.sleep(10)
                 res = con.read_until(wait.encode())

vjunosevolved/docker/init.conf

@@ -25,7 +25,10 @@ interfaces {
     re0:mgmt-0 {
         unit 0 {
             family inet {
-                address 10.0.0.15/24;
+                address {MGMT_IP_IPV4};
+            }
+            family inet {
+                address {MGMT_IP_IPV6};
             }
         }
     }
@@ -34,7 +37,12 @@ routing-instances {
     mgmt_junos {
         routing-options {
             static {
-                route 0.0.0.0/0 next-hop 10.0.0.2;
+                route 0.0.0.0/0 next-hop {MGMT_GW_IPV4};
+            }
+            rib mgmt_junos.inet6.0 {
+                static {
+                    route ::/0 next-hop {MGMT_GW_IPV6};
+                }
             }
         }
     }

vjunosevolved/docker/launch.py

@@ -55,6 +55,7 @@ def __init__(self, hostname, username, password, conn_mode):
             driveif="virtio",
             cpu="IvyBridge,vme=on,ss=on,vmx=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,bmi1=off,avx2=off,bmi2=off,erms=off,invpcid=off,rdseed=off,adx=off,smap=off,xsaveopt=off,abm=off,svm=off",
             smp="4,sockets=1,cores=4,threads=1",
+            mgmt_passthrough=True
         )
 
         # device hostname
@@ -67,16 +68,18 @@ def __init__(self, hostname, username, password, conn_mode):
         with open("init.conf", "r") as file:
             cfg = file.read()
 
-        # replace HOSTNAME file var with nodes given hostname
+        cfg = cfg.replace("{MGMT_IP_IPV4}", self.mgmt_address_ipv4)
+        cfg = cfg.replace("{MGMT_GW_IPV4}", self.mgmt_gw_ipv4)
+        cfg = cfg.replace("{MGMT_IP_IPV6}", self.mgmt_address_ipv6)
+        cfg = cfg.replace("{MGMT_GW_IPV6}", self.mgmt_gw_ipv6)
+        cfg = cfg.replace("{HOSTNAME}", self.hostname)
         # replace CRYPT_PSWD file var with nodes given password
         # (Evo does not accept plaintext passwords in config)
-        new_cfg = cfg.replace("{HOSTNAME}", hostname).replace(
-            "{CRYPT_PSWD}", password_hash
-        )
+        cfg = cfg.replace("{CRYPT_PSWD}", password_hash)
 
         # write changes to init.conf file
         with open("init.conf", "w") as file:
-            file.write(new_cfg)
+            file.write(cfg)
 
         # pass in user startup config
         self.startup_config()

vjunosrouter/docker/init.conf

@@ -25,7 +25,10 @@ interfaces {
     fxp0 {
         unit 0 {
             family inet {
-                address 10.0.0.15/24;
+                address {MGMT_IP_IPV4};
+            }
+            family inet6 {
+                address {MGMT_IP_IPV6};
             }
         }
     }
@@ -34,7 +37,12 @@ routing-instances {
     mgmt_junos {
         routing-options {
             static {
-                route 0.0.0.0/0 next-hop 10.0.0.2;
+                route 0.0.0.0/0 next-hop {MGMT_GW_IPV4};
+            }
+            rib mgmt_junos.inet6.0 {
+                static {
+                    route ::/0 next-hop {MGMT_GW_IPV6};
+                }
             }
         }
     }

vjunosrouter/docker/launch.py

@@ -52,6 +52,7 @@ def __init__(self, hostname, username, password, conn_mode):
             driveif="virtio",
             cpu="IvyBridge,vme=on,ss=on,vmx=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,bmi1=off,avx2=off,bmi2=off,erms=off,invpcid=off,rdseed=off,adx=off,smap=off,xsaveopt=off,abm=off,svm=on",
             smp="4,sockets=1,cores=4,threads=1",
+            mgmt_passthrough=True
         )
         # device hostname
         self.hostname = hostname
@@ -61,12 +62,15 @@ def __init__(self, hostname, username, password, conn_mode):
         with open("init.conf", "r") as file:
             cfg = file.read()
 
-        # replace HOSTNAME file var with nodes given hostname
-        new_cfg = cfg.replace("{HOSTNAME}", hostname)
+        cfg = cfg.replace("{MGMT_IP_IPV4}", self.mgmt_address_ipv4)
+        cfg = cfg.replace("{MGMT_GW_IPV4}", self.mgmt_gw_ipv4)
+        cfg = cfg.replace("{MGMT_IP_IPV6}", self.mgmt_address_ipv6)
+        cfg = cfg.replace("{MGMT_GW_IPV6}", self.mgmt_gw_ipv6)
+        cfg = cfg.replace("{HOSTNAME}", self.hostname)
 
         # write changes to init.conf file
         with open("init.conf", "w") as file:
-            file.write(new_cfg)
+            file.write(cfg)
 
         # pass in user startup config
         self.startup_config()