Dell OS10 improvements (#278)

* Add feature to wait while a holding pattern is detected

Needed for Dell OS10 "loading-OS10#" prompt

* Wait while prompt says "loading"...

* Add note on compression

* Remove note on "holding", no longer a concern

* Fix logging performance warnings

* Fix typo

* More performance issues

* * Make host/guest IPs configurable
* Make TCP ports to be forwarded configurable
* Remove Dell OS10 management IP changes - not needed (breaks Netlab provisioning)

* Don't change method signature

* Fix syntax; dynamic strings need explicit concatenation, compiler can't do it statically

* * Remove formatted log string
* Remove commented code block

common/vrnetlab.py

@@ -10,6 +10,7 @@
 import subprocess
 import telnetlib
 import time
+import ipaddress
 from pathlib import Path
 
 MAX_RETRIES = 60
@@ -114,6 +115,22 @@ def __init__(
         if min_dp_nics:
             self.min_nics = min_dp_nics
 
+        # management subnet properties, defaults
+        self.mgmt_subnet    = "10.0.0.0/24"
+        self.mgmt_host_ip   = 2
+        self.mgmt_guest_ip  = 15
+
+        #  Default TCP ports forwarded (TODO tune per platform):
+        #  80    - http
+        #  443   - https
+        #  830   - netconf
+        #  6030  - gnmi/gnoi arista
+        #  8080  - sonic gnmi/gnoi, other http apis
+        #  9339  - iana gnmi/gnoi
+        #  32767 - gnmi/gnoi juniper
+        #  57400 - nokia gnmi/gnoi
+        self.mgmt_tcp_ports = [80,443,830,6030,8080,9339,32767,57400]
+
         # we setup pci bus by default
         self.provision_pci_bus = provision_pci_bus
         self.nics_per_pci_bus = 26  # tested to work with XRv
@@ -294,9 +311,29 @@ def create_tc_tap_ifup(self):
         os.chmod("/etc/tc-tap-ifup", 0o777)
 
     def gen_mgmt(self):
-        """Generate qemu args for the mgmt interface(s)"""
+        """Generate qemu args for the mgmt interface(s)
+        
+        Default TCP ports forwarded:
+          80    - http
+          443   - https
+          830   - netconf
+          6030  - gnmi/gnoi arista
+          8080  - sonic gnmi/gnoi, other http apis
+          9339  - iana gnmi/gnoi
+          32767 - gnmi/gnoi juniper
+          57400 - nokia gnmi/gnoi
+        """
+        if self.mgmt_host_ip+1>=self.mgmt_guest_ip:
+            self.logger.error("Guest IP (%s) must be at least 2 higher than host IP(%s)", 
+                              self.mgmt_guest_ip, self.mgmt_host_ip)
+
+        network = ipaddress.ip_network(self.mgmt_subnet)
+        host = str(network[self.mgmt_host_ip])
+        dns = str(network[self.mgmt_host_ip+1])
+        guest = str(network[self.mgmt_guest_ip])
+
         res = []
-        # mgmt interface is special - we use qemu user mode network
+        # mgmt interface is special - we use qemu user mode network with DHCP
         res.append("-device")
         mac = (
             "c0:00:01:00:ca:fe"
@@ -306,18 +343,11 @@ def gen_mgmt(self):
         res.append(self.nic_type + f",netdev=p00,mac={mac}")
         res.append("-netdev")
         res.append(
-            "user,id=p00,net=10.0.0.0/24,"
-            "tftp=/tftpboot,"
-            "hostfwd=tcp:0.0.0.0:22-10.0.0.15:22,"  # ssh
-            "hostfwd=udp:0.0.0.0:161-10.0.0.15:161,"  # snmp
-            "hostfwd=tcp:0.0.0.0:830-10.0.0.15:830,"  # netconf
-            "hostfwd=tcp:0.0.0.0:80-10.0.0.15:80,"  # http
-            "hostfwd=tcp:0.0.0.0:443-10.0.0.15:443,"  # https
-            "hostfwd=tcp:0.0.0.0:9339-10.0.0.15:9339,"  # iana gnmi/gnoi
-            "hostfwd=tcp:0.0.0.0:57400-10.0.0.15:57400,"  # nokia gnmi/gnoi
-            "hostfwd=tcp:0.0.0.0:6030-10.0.0.15:6030,"  # gnmi/gnoi arista
-            "hostfwd=tcp:0.0.0.0:32767-10.0.0.15:32767,"  # gnmi/gnoi juniper
-            "hostfwd=tcp:0.0.0.0:8080-10.0.0.15:8080"  # sonic gnmi/gnoi, other http apis
+            f"user,id=p00,net={self.mgmt_subnet},host={host},dns={dns},dhcpstart={guest}," +
+            f"hostfwd=tcp:0.0.0.0:22-{guest}:22," +   # ssh
+            f"hostfwd=udp:0.0.0.0:161-{guest}:161," +  # snmp
+            (",".join([ f"hostfwd=tcp:0.0.0.0:{p}-{guest}:{p}" for p in self.mgmt_tcp_ports ])) +
+            ",tftp=/tftpboot"
         )
         return res
 
@@ -496,7 +526,7 @@ def restart(self):
         self.stop()
         self.start()
 
-    def wait_write(self, cmd, wait="__defaultpattern__", con=None, clean_buffer=False):
+    def wait_write(self, cmd, wait="__defaultpattern__", con=None, clean_buffer=False, hold=""):
         """Wait for something on the serial port and then send command
 
         Defaults to using self.tn as connection but this can be overridden
@@ -518,6 +548,12 @@ def wait_write(self, cmd, wait="__defaultpattern__", con=None, clean_buffer=Fals
             self.logger.trace(f"waiting for '{wait}' on {con_name}")
             res = con.read_until(wait.encode())
 
+            while (hold and (hold in res.decode())):
+                self.logger.trace(f"Holding pattern '{hold}' detected: {res.decode()}, retrying in 10s...")
+                con.write("\r".encode())
+                time.sleep(10)
+                res = con.read_until(wait.encode())
+
             cleaned_buf = (
                 (con.read_very_eager()) if clean_buffer else None
             )  # Clear any remaining characters in buffer

ftosv/README.md

@@ -200,10 +200,10 @@ Password:
 
 5. After the login (username: *admin*, password: *admin* or *linuxadmin*/*linuxadmin* followed by *su admin*), once the system is ready and the prompt appears, you need to **stop ztd** with the command `ztd cancel`. Then, `write memory` and `reload`.
 6. Once the reload is completed, you can shutdown the qemu-system host as the image has been built. With 10.5.2.4, it creates an image of approximately 7G.
-7. Next is to convert from vmdk to qcow2.
+7. Next is to convert from vmdk to qcow2 (tip: try adding "-c" for compression, may reduce qcow2 size to ~30%)
 
 ```bash
-qemu-img convert -f vmdk -O qcow2 OS10-Disk-1.0.0.vmdk dellftos.{VERSION}.qcow2
+qemu-img convert -f vmdk -O qcow2 -c OS10-Disk-1.0.0.vmdk dellftos.{VERSION}.qcow2
 ```
 
 Once this is complete, you'll be left with a qcow2 image that can then be built with the make command. To help validate output, here are the sizes of the 2 files.
@@ -233,14 +233,3 @@ NOTE:
 * CPU: 4 core
 * RAM: 4GB
 * Disk: <10GB
-
-## Caveats / Working vs Non-Working images
-
-Currently, the readiness check is performed by checking for the `OS10#` prompt after the login.
-
-Before version *10.5.6.1*, Dell OS10 allows you to login even if the system is still loading, displaying a `System is loading...` text, and showing the prompt only after the loading is completed.
-
-Apparently, starting from *10.5.6.1*, now you still can login even if the system is still loading, but you are presented a prompt in the format `loading-OS10#` - which, unfortunately, matches `OS10#`. If you launch any command on the *loading* prompt, you get an error.
-
-However, this **seems** to be solved again on *10.5.6.4*.
-

ftosv/docker/launch.py

@@ -6,6 +6,7 @@
 import re
 import signal
 import sys
+import ipaddress
 
 import vrnetlab
 
@@ -80,14 +81,22 @@ def __init__(self, hostname, username, password, conn_mode):
     def gen_mgmt(self):
         """
         Augment the parent class function to add gRPC port forwarding
+
+        TCP ports forwarded:
+        443 - OS10 REST API
+        830 - Netconf
         """
+        self.mgmt_subnet = "169.254.127.0/24"
+        self.mgmt_tcp_ports = [443,830]
         # call parent function to generate the mgmt interface
         res = super(FTOS_vm, self).gen_mgmt()
 
         # append gRPC forwarding if it was not added by common lib. confirm gNMI agent port number, default port is different than 50051?
         # gRPC Network Management Interface agent requires the switch in non default SmartFabric switch-operating-mode
-        if "hostfwd=tcp::50051-10.0.0.15:50051" not in res[-1]:
-            res[-1] = res[-1] + ",hostfwd=tcp::17051-10.0.0.15:50051"
+        network = ipaddress.ip_network(self.mgmt_subnet)
+        guest = str(network[self.mgmt_guest_ip])
+        if f"hostfwd=tcp::50051-{guest}:50051" not in res[-1]:
+            res[-1] = res[-1] + f",hostfwd=tcp::17051-{guest}:50051"
             vrnetlab.run_command(
                 ["socat", "TCP-LISTEN:50051,fork", "TCP:127.0.0.1:17051"],
                 background=True,
@@ -111,9 +120,7 @@ def bootstrap_spin(self):
                 except IndexError as exc:
                     self.logger.error("no more credentials to try")
                     return
-                self.logger.debug(
-                    "trying to log in with %s / %s" % (username, password)
-                )
+                self.logger.debug("trying to log in with %s / %s", username, password)
                 self.wait_write(username, wait=None)
                 self.wait_write(password, wait="Password:")
 
@@ -124,15 +131,15 @@ def bootstrap_spin(self):
                 self.tn.close()
                 # startup time?
                 startup_time = datetime.datetime.now() - self.start_time
-                self.logger.info("Startup complete in: %s" % startup_time)
+                self.logger.info("Startup complete in: %s", startup_time)
                 # mark as running
                 self.running = True
                 return
 
         # no match, if we saw some output from the router it's probably
         # booting, so let's give it some more time
         if res != b"":
-            self.logger.trace("OUTPUT: %s" % res.decode())
+            self.logger.trace("OUTPUT: %s", res.decode())
             # reset spins if we saw some output
             self.spins = 0
 
@@ -144,37 +151,31 @@ def bootstrap_config(self):
         """Do the actual bootstrap config"""
         self.logger.info("applying bootstrap configuration once system is ready")
         self.wait_write("", None)
-        self.wait_write("configure", wait="OS10#")
+        self.wait_write("configure", wait="OS10#", hold="loading-OS10#")
         self.wait_write(f"hostname {self.hostname}")
         self.wait_write("service simple-password")
         self.wait_write(
             f"username {self.username} password {self.password} role sysadmin priv-lv 15"
         )
 
-        # configure mgmt interface
-        self.wait_write("interface mgmt 1/1/1")
-        self.wait_write("no ip address dhcp")
-        self.wait_write("ip address 10.0.0.15/24")
-        self.wait_write("exit")
-        self.wait_write("management route 0.0.0.0/0 10.0.0.2")
-        self.wait_write("exit")
+        # No need to reconfigure mgmt IP; causes issues with parallel Netlab provisioning
         self.wait_write("copy running-configuration startup-configuration")
         self.wait_write("")
 
     def startup_config(self):
         """Load additional config provided by user."""
 
         if not os.path.exists(STARTUP_CONFIG_FILE):
-            self.logger.trace(f"Startup config file {STARTUP_CONFIG_FILE} is not found")
+            self.logger.trace("Startup config file %s is not found", STARTUP_CONFIG_FILE)
             return
 
-        self.logger.trace(f"Startup config file {STARTUP_CONFIG_FILE} exists")
+        self.logger.trace("Startup config file %s exists", STARTUP_CONFIG_FILE)
         with open(STARTUP_CONFIG_FILE) as file:
             config_lines = file.readlines()
             config_lines = [line.rstrip() for line in config_lines]
-            self.logger.trace(f"Parsed startup config file {STARTUP_CONFIG_FILE}")
+            self.logger.trace("Parsed startup config file %s", STARTUP_CONFIG_FILE)
 
-        self.logger.info(f"Writing lines from {STARTUP_CONFIG_FILE}")
+        self.logger.info("Writing lines from %s", STARTUP_CONFIG_FILE)
 
         self.wait_write("configure terminal")
         # Apply lines from file