Merge pull request #107 from sdgtt/dev

General updates

.codespell-whitelist

@@ -4,3 +4,4 @@ SOM
 som
 sinc
 ser
+hsi

.github/workflows/build.yml

@@ -0,0 +1,32 @@
+name: Nebula build test
+
+on: [push, pull_request]
+
+jobs:
+  Build-Test:
+    runs-on: ubuntu-latest
+    strategy:
+        matrix:
+            python-version: ["3.8", "3.9", "3.10"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+
+      - name: Install nebula
+        run: |
+          pip install .
+          python -c "import nebula"
+
+      - name: Test
+        run: |
+          cd tests
+          pytest test_cli.py::test_cli_help -s -v

.pre-commit-config.yaml

@@ -1,15 +1,15 @@
 repos:
 -   repo: https://github.com/codespell-project/codespell
-    rev: v1.15.0
+    rev: v2.2.6
     hooks:
     -   id: codespell
         args: [--ignore-words=.codespell-whitelist]
--   repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.4
+-   repo: https://github.com/pycqa/isort
+    rev: 5.13.2
     hooks:
-    -   id: isort
+      - id: isort
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.720
+    rev: v1.7.1
     hooks:
     -   id: mypy
         args: [--no-strict-optional, --ignore-missing-imports]

doc/manager_recovery.pu

@@ -0,0 +1,100 @@
+@startuml Nebula Manager Device Recovery
+start
+title Nebula Manager Device Recovery
+if (Is Linux is accessible?) then (yes)
+  partition "Verify IP Address" {
+    :(1)get ip address via uart,
+        if none assign one via dhcp
+     (2)if address is not expected,
+        update nebula config to new address;
+  }
+  #palegreen: Home sweet home;
+  kill
+else (no)
+
+  if (usbsdmux enabled) then (yes)
+    partition #yellow "SD Card Mux"{
+      : power down board using PDU
+      __power.power_down_board()__;
+      if (SD card mode) then (yes)
+        : update boot files
+        from within sdcard
+        via usb-sd-mux;
+      else (no)
+        : update boot files
+        from external sources
+        via usb-sd-mux;
+      endif
+      : Update device tree
+      via usb-sd-mux;
+    }
+    #Gray: Power cycle using PDU;
+    kill
+  else
+    if (JTAG enabled) then (yes)
+      : Restart board via JTAG;
+    else (no)
+    : Restart board via PDU;
+    endif
+
+    if (if uboot menu reached) then (yes)
+      if (if tftp enabled) then (yes)
+        : load boot files
+        from tftp via uart;
+        #Gray: Power cycle using PDU;
+        kill
+      else (no)
+        if (SD card mode) then (yes)
+          : copy bootfiles
+          from within sdcard via uart;
+        else (no)
+          : copy bootfiles
+          from external sources via uart;
+        endif
+        #Gray: Power cycle using PDU;
+        kill
+      endif
+    else (no)
+      #orange: Second Stage;
+      kill
+    endif
+  endif
+
+  #Gray: Power cycle using PDU;
+  if (Linux prompt reached) then (yes)
+    partition "Network check"{
+      if (is Network (IP) functional) then (yes)
+        if (is SSH  functional) then (yes)
+          #palegreen: Home sweet home;
+          kill
+        else (no)
+          #orange: Second Stage;
+          kill
+        endif
+      else (no)
+        #orange: Second Stage;
+        kill
+      endif
+    }
+  else (no)
+    #orange: Second Stage;
+    kill
+  endif
+  #orange :Second Stage;
+  : TO FIX JTAG Recovery
+  __manager.board_reboot_jtag_uart()__;
+  :__manager.power_cycle_to_boot()__;
+  if (power_cycle_to_boot ok) then (yes)
+    :__manager.network_check()__;
+    if (network check ok) then (yes)
+      #palegreen: Home sweet home;
+      kill
+    else (no)
+      #red: Recovery Failed;
+    endif
+  else (no)
+    #red: Recovery Failed;
+  endif
+endif
+end
+@enduml

nebula/__init__.py

@@ -3,22 +3,20 @@
 # print(f.renderText("Nebula"))
 import os
 
-from nebula.netconsole import netconsole
-from nebula.uart import uart
-from nebula.tftpboot import tftpboot
-from nebula.pdu import pdu
-from nebula.manager import manager
-from nebula.network import network
-from nebula.driver import driver
 from nebula.builder import builder
-from nebula.common import utils
-from nebula.helper import helper
-from nebula.downloader import downloader
+from nebula.common import LINUX_DEFAULT_PATH, utils
 from nebula.coverage import coverage
+from nebula.downloader import downloader
+from nebula.driver import driver
+from nebula.helper import helper
 from nebula.jtag import jtag
+from nebula.manager import manager
 from nebula.netbox import netbox
-
-from nebula.common import LINUX_DEFAULT_PATH
+from nebula.netconsole import netconsole
+from nebula.network import network
+from nebula.pdu import pdu
+from nebula.tftpboot import tftpboot
+from nebula.uart import uart
 
 if os.name in ["nt", "posix"] and os.path.exists(LINUX_DEFAULT_PATH):
     from nebula.usbmux import usbmux

nebula/builder.py

@@ -97,7 +97,7 @@ def linux_tools_map(self, branch, board):
             elif "2019_r1" in branch.lower() or "2019.1" in branch.lower():
                 vivado = "2018.3"
             elif branch == "master":
-                vivado = "2019.1"
+                vivado = "2021.2"
             else:
                 raise Exception("Unsupported branch")
         if "zcu102" in board.lower():

nebula/common.py

@@ -1,9 +1,10 @@
 import logging
 import os
 
-import nebula.errors as ne
 import yaml
 
+import nebula.errors as ne
+
 LINUX_DEFAULT_PATH = "/etc/default/nebula"
 WINDOWS_DEFAULT_PATH = "C:\\nebula\\nebula.yaml"
 

nebula/downloader.py

@@ -16,11 +16,12 @@
 from artifactory import ArtifactoryPath
 from bs4 import BeautifulSoup
 from github import Github
-from nebula.common import utils
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 from tqdm import tqdm
 
+from nebula.common import utils
+
 log = logging.getLogger(__name__)
 
 
@@ -283,6 +284,8 @@ def __init__(self, http_server_ip=None, yamlfilename=None, board_name=None):
         self.devicetree_overlay = None
         self.kernel = None
         self.modules = None
+        self.no_os_project = None
+        self.platform = None
         # update from config
         self.update_defaults_from_yaml(
             yamlfilename, __class__.__name__, board_name=board_name
@@ -671,6 +674,25 @@ def _get_files_rpi(
                 module_files = [tarinfo for tarinfo in tf.getmembers()]
             tf.extractall(path=dest, members=module_files)
 
+    def _get_files_noos(self, source, source_root, branch, project, platform):
+        dest = "outs"
+        if not os.path.isdir(dest):
+            os.mkdir(dest)
+        if source == "artifactory":
+            url_template = (
+                "https://{}/artifactory/sdg-generic-development/no-OS/{}/{}/{}/{}"
+            )
+            url = url_template.format(source_root, branch, "", "", "")
+            build_date = get_newest_folder(listFD(url))
+            url = url_template.format(
+                source_root, branch, build_date, platform, project + ".zip"
+            )
+            log.info(url)
+            file = os.path.join(dest, project + ".zip")
+            self.download(url, file)
+            # unzip the files
+            shutil.unpack_archive(file, dest)
+
     def _get_files(
         self,
         design_name,
@@ -686,6 +708,8 @@ def _get_files(
         devicetree_overlay,
         kernel,
         modules,
+        noos_project,
+        platform,
         folder=None,
         firmware=False,
         noos=False,
@@ -718,6 +742,8 @@ def _get_files(
         elif "RPI" in details["carrier"]:
             kernel = kernel
             modules = modules
+        elif details["carrier"] in ["Maxim", "ADICUP"]:
+            pass
         else:
             raise Exception("Carrier not supported")
 
@@ -738,8 +764,8 @@ def _get_files(
                 # design_source_root = os.path.join(source_root, design_name)
 
             if noos:
-                self._get_files_hdl(
-                    hdl_folder, source, source_root, branch, hdl_output=True
+                self._get_files_noos(
+                    source, source_root, branch, noos_project, platform
                 )
 
             if microblaze:
@@ -845,24 +871,10 @@ def download_boot_files(
         devicetree_overlay = self.devicetree_overlay
         kernel = self.kernel
         modules = self.modules
+        noos_project = self.no_os_project
+        platform = self.platform
 
-        if noos:
-            res = os.path.join(path, "resources", "noOS_projects.yaml")
-            with open(res) as f:
-                noos_projects = yaml.load(f, Loader=yaml.FullLoader)
-            val = []
-            for project in noos_projects:
-                hdl_projects = noos_projects[project]
-                if hdl_projects is not None:
-                    for hdl_project in hdl_projects:
-                        if hdl_project == hdl_folder:
-                            val.append(hdl_project)
-                            log.info("No-OS project:" + project)
-
-            if not val:
-                raise Exception("Design has no support!")
-        else:
-            assert design_name in board_configs, "Invalid design name"
+        assert design_name in board_configs, "Invalid design name"
 
         if not firmware:
             matched = re.match("v[0-1].[0-9][0-9]", branch)
@@ -893,6 +905,8 @@ def download_boot_files(
             devicetree_overlay,
             kernel,
             modules,
+            noos_project,
+            platform,
             folder,
             firmware,
             noos,

nebula/errors.py

@@ -30,7 +30,13 @@ class SSHNotFunctionalAfterBootFileUpdate(Error):
 
 
 class LinuxNotReached(Error):
-    """Linux is accessible (likely previous bad BOOT.BIN or kernel crash)"""
+    """Linux is inaccessible (likely previous bad BOOT.BIN or kernel crash)"""
+
+    pass
+
+
+class UbootNotReached(Error):
+    """U-boot menu not reachable"""
 
     pass