Merge pull request #114 from sdgtt/dev-plug-and-test

plug and test feature

nebula/helper.py

@@ -1,7 +1,10 @@
 import glob
+import json
 import logging
 import os
 import pathlib
+import re
+from functools import partial
 
 import click
 import netifaces
@@ -77,6 +80,16 @@ def get_nics():
     return (str, default)
 
 
+def project_filter(project_dict, filters):
+    match = True
+    for k, v in filters.items():
+        if k in project_dict.keys():
+            if not project_dict[k] == v:
+                match = False
+                break
+    return match
+
+
 class helper:
     def __init__(self):
         pass
@@ -342,3 +355,172 @@ def _write_config_file(self, filename, outconfig):
         file1 = open(filename, "w")
         file1.writelines(lines)
         file1.close()
+
+    def get_boot_files_from_descriptor(self, descriptor_file, project):
+        """
+        Extracts the project bootfiles defined on the kuiper desctriptor file.
+        i.e kuiper.json
+        """
+
+        boot_files = (
+            list()
+        )  # contains all files needed to be moved to the boot partition
+
+        common_architectures = [
+            ("arria10_", "arria10"),
+            ("cyclone5_", "cyclone5"),
+            ("zynq-", "zynq"),
+            ("zynq-", "zynq"),
+            ("zynqmp-", "zynqmp"),
+            ("versal-", "versal"),
+        ]
+        common_boards = [
+            ("socdk_", "socdk"),
+            ("de10_nano_", "de10nano"),
+            ("sockit_", "sockit"),
+            ("coraz7s-", "coraz7s"),
+            ("zc702-", "zc702"),
+            ("zc706-", "zc706"),
+            ("zed-", "zed"),
+            ("zcu102-", "zcu102"),
+            ("adrv9009-zu11eg-", "adrv9009zu11eg_adrv2crr"),
+            ("vck190-", "vck190"),
+            ("-bob", "ccbob"),
+            ("z7035-fmc", "ccfmc"),
+            ("z7035-packrf", "ccpackrf"),
+            ("z7020-packrf", "ccpackrf"),
+        ]
+        common_names = [
+            ("ad9081$", "ad9081"),
+            ("adv7511$", "adv7511"),
+            ("ad9695", "ad9695"),
+            ("ad9783", "ad9783"),
+            ("adrv9002$", "adrv9002"),
+            ("adrv9009", "adrv9009"),
+            ("adrv9371", "adrv9371"),
+            ("adrv9375", "adrv9375"),
+            ("cn0540", "cn0540"),
+            ("cn0579", "cn0579"),
+            ("_daq2", "daq2"),
+            ("fmcdaq2", "daq2"),
+            ("fmcdaq3", "daq3"),
+            ("fmcadc2", "fmcadc2"),
+            ("fmcadc3", "fmcadc3"),
+            ("fmcjesdadc1", "fmcjesdadc1"),
+            ("fmcomms11", "fmcomms11"),
+            ("fmcomms2", "fmcomms2"),
+            ("fmcomms3", "fmcomms3"),
+            ("fmcomms4", "fmcomms4"),
+            ("fmcomms5", "fmcomms5"),
+            ("fmcomms8", "fmcomms8"),
+            ("cn0501", "cn0501"),
+            ("ad4020", "ad4020"),
+            ("cn0363", "cn0363"),
+            ("cn0577", "cn0577"),
+            ("imageon", "imageon"),
+            ("ad4630-24", "ad4630_fmc"),
+            ("ad7768-axi-adc", "ad7768"),
+            ("ad7768-1-evb", "ad77681_evb"),
+            ("ad7768-4-axi-adc", "ad7768-4"),
+            ("adaq8092", "adaq8092"),
+            ("socdk_fmclidar1", "ad_fmclidar1_ebz"),
+            ("adv7511-fmclidar1", "ad_fmclidar1_ebz"),
+            ("rev10-fmclidar1", "fmclidar"),
+            ("adrv9002[-_]rx2tx2", "adrv9002_rx2tx2"),
+            ("cn0506[-_]mii", "cn0506_mii"),
+            ("cn0506[-_]rgmii", "cn0506_rgmii"),
+            ("cn0506[-_]rmii", "cn0506_rmii"),
+            ("ad6676-fmc", "ad6676evb"),
+            ("ad9265-fmc-125ebz", "ad9265_fmc"),
+            ("ad9434-fmc", "ad9434_fmc"),
+            ("ad9739a-fmc", "ad9739a_fmc"),
+            ("adrv9008-1", "adrv9008-1"),
+            ("adrv9008-2", "adrv9008-2"),
+            ("ad9172-fmc-ebz", "ad9172_fmc"),
+            ("fmcomms5-ext-lo-adf5355", "fmcomms5-ext-lo-adf5355"),
+            ("z7035-bob-vcmos", "adrv9361z7035_cmos"),
+            ("z7035-bob-vlvds", "adrv9361z7035_lvds"),
+            ("z7020-bob-vcmos", "adrv9364z7020_cmos"),
+            ("z7020-bob-vlvds", "adrv9364z7020_lvds"),
+            ("z7035-fmc", "adrv9361z7035_lvds"),
+            ("z7035-packrf", "adrv9361z7035_lvds"),
+            ("z7020-packrf", "adrv9364z7020_lvds"),
+            ("ad9467-fmc-250ebz", "ad9467-fmc"),
+            ("otg", "adv7511_without_bitstream"),
+            ("hps", "de10nano_without_bitfile"),
+            ("adrv2crr-fmc-revb", "adrv9009zu11eg_adrv2crr"),
+            ("multisom-primary", "multisom-primary"),
+            ("multisom-secondary", "multisom-secondary"),
+            ("fmcomms8-multisom-primary", "fmcomms8_multisom_primary"),
+            ("fmcomms8-multisom-secondary", "fmcomms8_multisom_secondary"),
+            ("xmicrowave", "xmicrowave"),
+            ("ad9081-vm8-l4", "ad9081_m8_l4"),
+            ("ad9081-vm4-l8", "ad9081_m4_l8"),
+            ("ad9081[-_]vnp12", "ad9081_np12"),
+            ("ad9081-vm8-l4-vcxo122p88", "ad9081_m8_l4_vcxo122p88"),
+            ("ad9081-v204b-txmode9-rxmode4", "ad9081_204b_txmode9_rxmode4"),
+            ("ad9081-v204c-txmode0-rxmode1", "ad9081_204c_txmode0_rxmode1"),
+            ("ad9082-m4-l8", "ad9082_m4_l8"),
+            ("ad9082$", "ad9082"),
+            ("ad9083-fmc-ebz", "ad9083"),
+            ("adrv9008-1", "adrv9008-1"),
+            ("adrv9008-2", "adrv9008-2"),
+            ("adv7511-adrv9002-vcmos", "adrv9002"),
+            ("rev10-adrv9002-vcmos", "adrv9002_cmos"),
+            ("rev10-adrv9002-vlvds", "adrv9002_lvds"),
+            ("adv7511-adrv9002-rx2tx2-vcmos", "adrv9002_rx2tx2"),
+            ("rev10-adrv9002-rx2tx2-vcmos", "adrv9002_rx2tx2_cmos"),
+            ("rev10-adrv9002-rx2tx2-vlvds", "adrv9002_rx2tx2_lvds"),
+            ("ad9172-fmc-ebz-mode4", "ad9172_mode4"),
+            ("arradio", "sockit_arradio"),
+            ("adrv9025", "adrv9025"),
+        ]
+
+        with open(descriptor_file, "r") as f:
+            descriptor = json.load(f)
+
+        assert descriptor
+
+        # for platform in
+
+        p_architecture = None
+        p_board = None
+        p_name = None
+
+        for ar in common_architectures:
+            if re.search(ar[0], project):
+                p_architecture = ar[1]
+
+        for br in common_boards:
+            if re.search(br[0], project):
+                p_board = br[1]
+
+        for pn in common_names:
+            if re.search(pn[0], project):
+                p_name = pn[1]
+
+        projects = descriptor["projects"]
+
+        # filter project
+        if project:
+            filter_dict = dict(
+                {"architecture": p_architecture, "board": p_board, "name": p_name}
+            )
+            projects = filter(partial(project_filter, filters=filter_dict), projects)
+
+        for project in projects:
+            # if not project['kernel'] in [ bt[1] for bt in boot_files]:
+            # boot_files.append((project['name'],project['kernel']))
+            boot_files.append((project["name"], project["kernel"]))
+            if "preloader" in project:
+                boot_files.append((project["name"], project["preloader"]))
+            files = project["files"]
+            for f in files:
+                boot_files.append((project["name"], f["path"]))
+
+        # check if project is supported
+        log.info("path:" + str(boot_files))
+        if not boot_files:
+            raise Exception("Project not supported in this nebula version.")
+
+        return boot_files

nebula/manager.py

@@ -431,17 +431,19 @@ def board_reboot_uart_net_pdu(
         extlinux_path=None,
         scr_path=None,
         preloader_path=None,
+        sdcard=False,
     ):
         """Manager when UART, PDU, and Network are available"""
-        self._check_files_exist(
-            system_top_bit_path,
-            bootbinpath,
-            uimagepath,
-            devtreepath,
-            extlinux_path,
-            scr_path,
-            preloader_path,
-        )
+        if not sdcard:
+            self._check_files_exist(
+                system_top_bit_path,
+                bootbinpath,
+                uimagepath,
+                devtreepath,
+                extlinux_path,
+                scr_path,
+                preloader_path,
+            )
         try:
             # Flush UART
             self.monitor[0]._read_until_stop()  # Flush
@@ -475,14 +477,17 @@ def board_reboot_uart_net_pdu(
 
             # Update board over SSH and reboot
             log.info("Update board over SSH and reboot")
-            self.net.update_boot_partition(
-                bootbinpath=bootbinpath,
-                uimagepath=uimagepath,
-                devtreepath=devtreepath,
-                extlinux_path=extlinux_path,
-                scr_path=scr_path,
-                preloader_path=preloader_path,
-            )
+            if sdcard:
+                self.net.update_boot_partition_existing_files(self.board_name)
+            else:
+                self.net.update_boot_partition(
+                    bootbinpath=bootbinpath,
+                    uimagepath=uimagepath,
+                    devtreepath=devtreepath,
+                    extlinux_path=extlinux_path,
+                    scr_path=scr_path,
+                    preloader_path=preloader_path,
+                )
             log.info("Waiting for reboot to complete")
 
             # Verify uboot anad linux are reached
@@ -550,6 +555,7 @@ def board_reboot_sdmux_pdu(
         extlinux_path=None,
         scr_path=None,
         preloader_path=None,
+        sdcard=False,
     ):
         """Manager when sdcardmux, pdu is available"""
 
@@ -585,16 +591,19 @@ def board_reboot_sdmux_pdu(
                 )
 
             log.info("Update board over usb-sd-mux")
-            self.usbsdmux.update_boot_files_from_external(
-                bootbin_loc=bootbinpath,
-                kernel_loc=uimagepath,
-                devicetree_loc=devtreepath,
-                devicetree_overlay_loc=devtree_overlay_path,
-                devicetree_overlay_config_loc=devtree_overlay_config_path,
-                extlinux_loc=extlinux_path,
-                scr_loc=scr_path,
-                preloader_loc=preloader_path,
-            )
+            if sdcard:
+                self.usbsdmux.update_boot_files_from_sdcard_itself()
+            else:
+                self.usbsdmux.update_boot_files_from_external(
+                    bootbin_loc=bootbinpath,
+                    kernel_loc=uimagepath,
+                    devicetree_loc=devtreepath,
+                    devicetree_overlay_loc=devtree_overlay_path,
+                    devicetree_overlay_config_loc=devtree_overlay_config_path,
+                    extlinux_loc=extlinux_path,
+                    scr_loc=scr_path,
+                    preloader_loc=preloader_path,
+                )
             # if devtreepath:
             #     self.usbsdmux.update_devicetree_for_mux(devtreepath)
             self.usbsdmux.set_mux_mode("dut")
@@ -822,6 +831,7 @@ def board_reboot_auto_folder(
                     system_top_bit_path=bit,
                     uimagepath=kernel,
                     devtreepath=dt,
+                    sdcard=sdcard,
                 )
             else:
                 self.board_reboot_auto(
@@ -873,6 +883,7 @@ def board_reboot_auto(
                         extlinux_path=extlinux_path,
                         scr_path=scr_path,
                         preloader_path=preloader_path,
+                        sdcard=sdcard,
                     )
                 else:
                     raise Exception("SD Card Mux not Supported")
@@ -887,6 +898,7 @@ def board_reboot_auto(
                     extlinux_path=extlinux_path,
                     scr_path=scr_path,
                     preloader_path=preloader_path,
+                    sdcard=sdcard,
                 )
 
     def shutdown_powerdown_board(self):

nebula/network.py

@@ -12,6 +12,7 @@
 from fabric import Connection
 
 import nebula.errors as ne
+import nebula.helper as helper
 from nebula.common import utils
 
 log = logging.getLogger(__name__)
@@ -272,20 +273,42 @@ def update_boot_partition_existing_files(self, subfolder=None):
         log.info("Updating boot files over SSH from SD card itself")
         if not subfolder:
             raise Exception("Must provide a subfolder")
-        self.run_ssh_command("mkdir /tmp/sdcard")
-        self.run_ssh_command("mount /dev/mmcblk0p1 /tmp/sdcard")
-        self.run_ssh_command("cp /tmp/sdcard/" + subfolder + "/BOOT.BIN /tmp/sdcard/")
-        if "zynqmp" in subfolder:
-            self.run_ssh_command("cp /tmp/sdcard/zynqmp-common/Image /tmp/sdcard/")
-            self.run_ssh_command(
-                "cp /tmp/sdcard/" + subfolder + "/system.dtb /tmp/sdcard/"
-            )
+        log.info("Updating boot files over SSH")
+        try:
+            self.run_ssh_command("ls /tmp/sdcard", retries=1)
+            dir_exists = True
+        except Exception:
+            log.info("Existing /tmp/sdcard directory not found. Will need to create it")
+            dir_exists = False
+        if dir_exists:
+            try:
+                log.info("Trying to unmounting directory")
+                self.run_ssh_command("umount /tmp/sdcard", retries=1)
+            except Exception:
+                log.info("Unmount failed... Likely not mounted")
+                pass
         else:
-            self.run_ssh_command("cp /tmp/sdcard/zynq-common/uImage /tmp/sdcard/")
-            self.run_ssh_command(
-                "cp /tmp/sdcard/" + subfolder + "/devicetree.dtb /tmp/sdcard/"
-            )
-        self.run_ssh_command("sudo reboot")
+            self.run_ssh_command("mkdir /tmp/sdcard", retries=1)
+
+        self.run_ssh_command("mount /dev/mmcblk0p1 /tmp/sdcard")
+
+        # extract needed boot files from the kuiper descriptor file
+        h = helper()
+        path = pathlib.Path(__file__).parent.absolute()
+        descriptor_path = os.path.join(path, "resources", "kuiper.json")
+        try:
+            self._dl_file("/tmp/sdcard/kuiper.json")
+            descriptor_path = "kuiper.json"
+        except Exception:
+            log.warning("Cannot find project descriptor on target")
+
+        boot_files_path = h.get_boot_files_from_descriptor(descriptor_path, subfolder)
+
+        for boot_file in boot_files_path:
+            log.info(f"Copying {boot_file[1]}")
+            self.run_ssh_command(f"cp {boot_file[1]} /tmp/sdcard/")
+
+        self.run_ssh_command("sudo reboot", ignore_exceptions=True)
 
     def _dl_file(self, filename):
         fabric.Connection(