Merge pull request #101 from sdgtt/dl-no-os

downloader: update no-os file downloader

nebula/downloader.py

@@ -283,6 +283,8 @@ def __init__(self, http_server_ip=None, yamlfilename=None, board_name=None):
         self.devicetree_overlay = None
         self.kernel = None
         self.modules = None
+        self.no_os_project = None
+        self.platform = None
         # update from config
         self.update_defaults_from_yaml(
             yamlfilename, __class__.__name__, board_name=board_name
@@ -671,6 +673,25 @@ def _get_files_rpi(
                 module_files = [tarinfo for tarinfo in tf.getmembers()]
             tf.extractall(path=dest, members=module_files)
 
+    def _get_files_noos(self, source, source_root, branch, project, platform):
+        dest = "outs"
+        if not os.path.isdir(dest):
+            os.mkdir(dest)
+        if source == "artifactory":
+            url_template = (
+                "https://{}/artifactory/sdg-generic-development/no-OS/{}/{}/{}/{}"
+            )
+            url = url_template.format(source_root, branch, "", "", "")
+            build_date = get_newest_folder(listFD(url))
+            url = url_template.format(
+                source_root, branch, build_date, platform, project + ".zip"
+            )
+            log.info(url)
+            file = os.path.join(dest, project + ".zip")
+            self.download(url, file)
+            # unzip the files
+            shutil.unpack_archive(file, dest)
+
     def _get_files(
         self,
         design_name,
@@ -686,6 +707,8 @@ def _get_files(
         devicetree_overlay,
         kernel,
         modules,
+        noos_project,
+        platform,
         folder=None,
         firmware=False,
         noos=False,
@@ -718,6 +741,8 @@ def _get_files(
         elif "RPI" in details["carrier"]:
             kernel = kernel
             modules = modules
+        elif details["carrier"] in ["Maxim", "ADICUP"]:
+            pass
         else:
             raise Exception("Carrier not supported")
 
@@ -738,8 +763,8 @@ def _get_files(
                 # design_source_root = os.path.join(source_root, design_name)
 
             if noos:
-                self._get_files_hdl(
-                    hdl_folder, source, source_root, branch, hdl_output=True
+                self._get_files_noos(
+                    source, source_root, branch, noos_project, platform
                 )
 
             if microblaze:
@@ -845,24 +870,10 @@ def download_boot_files(
         devicetree_overlay = self.devicetree_overlay
         kernel = self.kernel
         modules = self.modules
+        noos_project = self.no_os_project
+        platform = self.platform
 
-        if noos:
-            res = os.path.join(path, "resources", "noOS_projects.yaml")
-            with open(res) as f:
-                noos_projects = yaml.load(f, Loader=yaml.FullLoader)
-            val = []
-            for project in noos_projects:
-                hdl_projects = noos_projects[project]
-                if hdl_projects is not None:
-                    for hdl_project in hdl_projects:
-                        if hdl_project == hdl_folder:
-                            val.append(hdl_project)
-                            log.info("No-OS project:" + project)
-
-            if not val:
-                raise Exception("Design has no support!")
-        else:
-            assert design_name in board_configs, "Invalid design name"
+        assert design_name in board_configs, "Invalid design name"
 
         if not firmware:
             matched = re.match("v[0-1].[0-9][0-9]", branch)
@@ -893,6 +904,8 @@ def download_boot_files(
             devicetree_overlay,
             kernel,
             modules,
+            noos_project,
+            platform,
             folder,
             firmware,
             noos,

nebula/resources/board_table.yaml

@@ -475,3 +475,13 @@ dc1962c:
   carrier: RPI
 dc2873a:
   carrier: RPI
+
+# ADD no-OS boards
+max32650_adxl355:
+  carrier: Maxim
+max78000_adt7420:
+  carrier: Maxim
+aducm3029_adxrs290:
+  carrier: ADICUP
+vc707_fmcomms2:
+  carrier: VC707

nebula/resources/template_noos_gen.yaml

@@ -0,0 +1,217 @@
+# This file is used to generate questions for configuration file generation
+# Here is the structure
+# <section>-config:
+#   field_X:
+#     name: <field name>
+#     default: <default option> (Optional)
+#     help: <question asked user>
+#     options: <list of options> (Optional)
+#     optional: <False,True,depends> (depends is used when its a dependent property)
+#     requires: <answer needed>:<field name of fields depending on this one,second field name> (Optional)
+#     netbox_field: <equivalent netbox object attribute>
+
+board-config:
+  field_1:
+    name: board-name
+    default: max32650_adxl355
+    help: "Project name commonly the daughter board connected to an RPI-based carrier"
+    optional: False
+    netbox_field: devices.name
+  field_2:
+    name: carrier
+    default: Maxim
+    help: "Carrier board name"
+    optional: True
+    netbox_field: devices.custom_fields.device_carrier
+  field_3:
+    name: daughter
+    help: "Daugther board name"
+    optional: True
+    netbox_field: devices.custom_fields.device_daughter
+  field_4:
+    name: monitoring-interface
+    default: uart
+    help: "Select monitoring interface"
+    options: [uart, netconsole]
+    optional: True
+    netbox_field: devices.custom_fields.monitoring_interface
+  field_5:
+    name: example
+    default: iio_example
+    help: "Select example to load to target"
+    optional: False
+    netbox_field: devices.custom_fields.example
+network-config:
+  field_1:
+    name: dutip
+    default: 192.168.10.2
+    help: "IP address of development board"
+    optional: False
+    netbox_field: devices.interfaces.mgmt.ip.address
+  field_2:
+    name: dhcp
+    help: "DHCP network to development board (False assumes static)"
+    optional: True
+    options: ["True", "False"]
+    requires: False:nic,nicip
+    netbox_field: devices.interfaces.mgmt.custom_fields.dhcp
+  field_3:
+    name: nic
+    default: eth0
+    help: "NIC used to talk to development board"
+    optional: depends
+    callback: get_nics
+    netbox_field: devices.interfaces.mgmt.label
+  field_4:
+    name: nicip
+    default: 192.168.10.1
+    help: "NIC connected to development board ip address"
+    optional: depends
+    netbox_field: devices.interfaces.mgmt.custom_fields.nicip
+  field_5:
+    name: dutusername
+    default: analog
+    help: "Username to access Device"
+    optional: True
+    netbox_field: devices.custom_fields.username
+  field_6:
+    name: dutpassword
+    default: analog
+    help: "Password associated with the username."
+    optional: True
+    netbox_field: devices.custom_fields.password
+pdu-config:
+  field_1:
+    name: pdu_type
+    default: cyberpower
+    help: "PDU type"
+    options: [cyberpower, vesync]
+    optional: True
+    requires: vesync:username,password
+    netbox_field: devices.power_ports.input.pdus[0].device_type.manufacturer.slug
+  field_2:
+    name: pduip
+    default: 192.168.30.2
+    help: "IP address of PDU"
+    optional: True
+    netbox_field: devices.power_ports.input.pdus[0].primary_ip.address
+  field_3:
+    name: outlet
+    default: 1
+    help: "Outlet number on PDU for dev board"
+    optional: True
+    type: int
+    netbox_field: devices.power_ports.input.connected_endpoint.outlet
+  field_4:
+    name: username
+    default: "username"
+    help: "Username needed for login (only need for vesync so far)"
+    optional: depends
+    netbox_field: devices.power_ports.input.pdus[0].custom_fields.username
+  field_5:
+    name: password
+    default: "password"
+    help: "Password needed for login (only need for vesync so far)"
+    optional: depends
+    netbox_field: devices.power_ports.input.pdus[0].custom_fields.password
+uart-config:
+  field_1:
+    name: address
+    help: "UART Address"
+    optional: False
+    callback: get_uarts
+    netbox_field: devices.console_ports.UART.label
+  field_2:
+    name: baudrate
+    help: "UART baudrate"
+    optional: False
+    netbox_field: devices.console_ports.UART.custom_fields.baudrate
+  field_3:
+    name: logfilename
+    default: device.log
+    help: "Output UART logfilename"
+    optional: True
+    netbox_field: devices.custom_fields.logfilename
+system-config:
+  field_1:
+    name: tftpserverip
+    help: "TFTP server address"
+    optional: True
+    netbox_field: devices.config_context.tftpserverip
+  field_2:
+    name: tftpserverroot
+    help: "TFTP folder location"
+    optional: True
+    netbox_field: devices.config_context.tftpserverroot
+driver-config:
+  field_1:
+    name: iio_device_names
+    default: ["adxl355"]
+    help: "List of IIO devices on board"
+    optional: False
+    netbox_field: devices.custom_fields.iio_device_names
+downloader-config:
+  field_1:
+    name: http_server_ip
+    help: "IP address of build server with boot files"
+    optional: True
+  field_2:
+    name: no_os_project
+    help: "Project folder name in no-os repo"
+    optional: False
+    netbox_field: devices.custom_fields.no_os_project
+  field_3:
+    name: platform
+    help: "Platform used of the no-os project"
+    optional: False
+    netbox_field: devices.custom_fields.platform
+netbox-config:
+  field_1:
+    name: netbox_server
+    help: "IP address of the netbox server"
+    default: 192.168.10.1
+    optional: True
+    netbox_field: devices.config_context.netbox_server
+  field_2:
+    name: netbox_server_port
+    default: 8000
+    help: "Port used by the netbox service"
+    optional: True
+    netbox_field: devices.config_context.netbox_port
+  field_3:
+    name: netbox_base_url
+    default: netbox
+    help: "String used as netbox base url. i.e server:port\base_url"
+    optional: True
+    netbox_field: devices.config_context.netbox_base_url
+  field_4:
+    name: netbox_api_token
+    default: 0123456789abcdef0123456789abcdef01234567
+    help: "Token for netbox rest api access"
+    optional: True
+    netbox_field: devices.config_context.netbox_api_token
+jtag-config:
+  field_1:
+    name: vivado_version
+    default: 2019.1
+    help: "Version of vivado to use"
+    optional: True
+    netbox_field: devices.config_context.vivado_version
+  field_2:
+    name: custom_vivado_path
+    default: None
+    help: "Custom path to vivado including version.\nEx: /opt/Xilinx/Vivado/2019.1\nOverrides vivado_version if set"
+    optional: True
+    netbox_field: devices.config_context.custom_vivado_path
+  field_3:
+    name: jtag_cable_id
+    default: 210299A567FE
+    help: "Substring of JTAG cable ID. Run 'jtag target' through xsdb to get it.\n Just really need code not full name."
+    optional: True
+    netbox_field: devices.console_ports.JTAG.label
+  field_4:
+    name: jtag_cpu_target_name
+    default: ARM*#0
+    help: "Name use to identify jtag target\n. This will be used for filtering jtag target.\n Can use wildcards e.x *"
+    optional: True
+    netbox_field: devices.custom_fields.jtag_cpu_target_name

setup.py

@@ -37,6 +37,7 @@ def _get_version():
             "resources/noOS_projects.yaml",
             "resources/template_agent_gen.yaml",
             "resources/template_rpi_gen.yaml",
+            "resources/template_noos_gen.yaml",
         ]
     },
     include_package_data=True,

tests/nebula_config/nebula.yaml

@@ -1,21 +1,32 @@
+max32650_adxl355:
+  board-config:
+  - board-name: max32650_adxl355
+  - carrier: Maxim
+  - daughter: ADXL355
+  - monitoring-interface: uart
+  - example: iio_example
+  downloader-config:
+  - no_os_project: eval-adxl355-pmdz
+  - platform: Maxim
 zynq-zc702-adv7511-ad9361-fmcomms2-3:
   board-config:
-    - board-name: zynq-zc702-adv7511-ad9361-fmcomms2-3
-    - monitoring_interface: uart
-    - allow_jtag: True
-    - no-os-project: ad9361
+  - board-name: zynq-zc702-adv7511-ad9361-fmcomms2-3
+  - monitoring_interface: uart
+  - allow_jtag: True
   downloader-config:
-    - reference_boot_folder: zynq-zc702-adv7511-ad936x-fmcomms2-3-4
-    - devicetree_subfolder: zynq-zc702-adv7511-ad9361-fmcomms2-3
-    - hdl_folder: fmcomms2_zc702
+  - reference_boot_folder: zynq-zc702-adv7511-ad936x-fmcomms2-3-4
+  - devicetree_subfolder: zynq-zc702-adv7511-ad9361-fmcomms2-3
+  - hdl_folder: fmcomms2_zc702
+  - no_os_project: ad9361
+  - platform: Xilinx
   driver-config:
-    - iio_device_names:
-      - ad7291
-      - ad9361-phy
-      - adf4351-udc-tx-pmod
-      - adf4351-udc-rx-pmod
-      - cf-ad9361-dds-core-lpc
-      - cf-ad9361-lpc
+  - iio_device_names:
+    - ad7291
+    - ad9361-phy
+    - adf4351-udc-tx-pmod
+    - adf4351-udc-rx-pmod
+    - cf-ad9361-dds-core-lpc
+    - cf-ad9361-lpc
 zynq-zc706-adv7511-fmcomms11:
   board-config:
   - board-name: zynq-zc706-adv7511-fmcomms11