diff --git a/.codespell-whitelist b/.codespell-whitelist index 1e1aaf9e..16467808 100644 --- a/.codespell-whitelist +++ b/.codespell-whitelist @@ -4,3 +4,4 @@ SOM som sinc ser +hsi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2cc43dc0..659437d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,15 @@ repos: - repo: https://github.com/codespell-project/codespell - rev: v1.15.0 + rev: v2.2.6 hooks: - id: codespell args: [--ignore-words=.codespell-whitelist] -- repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.4 +- repo: https://github.com/pycqa/isort + rev: 5.13.2 hooks: - - id: isort + - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.720 + rev: v1.7.1 hooks: - id: mypy args: [--no-strict-optional, --ignore-missing-imports] diff --git a/nebula/__init__.py b/nebula/__init__.py index f9bf1ed9..187ca68f 100644 --- a/nebula/__init__.py +++ b/nebula/__init__.py @@ -3,22 +3,20 @@ # print(f.renderText("Nebula")) import os -from nebula.netconsole import netconsole -from nebula.uart import uart -from nebula.tftpboot import tftpboot -from nebula.pdu import pdu -from nebula.manager import manager -from nebula.network import network -from nebula.driver import driver from nebula.builder import builder -from nebula.common import utils -from nebula.helper import helper -from nebula.downloader import downloader +from nebula.common import LINUX_DEFAULT_PATH, utils from nebula.coverage import coverage +from nebula.downloader import downloader +from nebula.driver import driver +from nebula.helper import helper from nebula.jtag import jtag +from nebula.manager import manager from nebula.netbox import netbox - -from nebula.common import LINUX_DEFAULT_PATH +from nebula.netconsole import netconsole +from nebula.network import network +from nebula.pdu import pdu +from nebula.tftpboot import tftpboot +from nebula.uart import uart if os.name in ["nt", "posix"] and os.path.exists(LINUX_DEFAULT_PATH): from nebula.usbmux import usbmux diff --git a/nebula/common.py b/nebula/common.py index a2744245..ce80c5c1 100644 --- a/nebula/common.py +++ b/nebula/common.py @@ -1,9 +1,10 @@ import logging import os -import nebula.errors as ne import yaml +import nebula.errors as ne + LINUX_DEFAULT_PATH = "/etc/default/nebula" WINDOWS_DEFAULT_PATH = "C:\\nebula\\nebula.yaml" diff --git a/nebula/downloader.py b/nebula/downloader.py index 5f790614..a11c643e 100644 --- a/nebula/downloader.py +++ b/nebula/downloader.py @@ -16,11 +16,12 @@ from artifactory import ArtifactoryPath from bs4 import BeautifulSoup from github import Github -from nebula.common import utils from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry from tqdm import tqdm +from nebula.common import utils + log = logging.getLogger(__name__) diff --git a/nebula/fixtures.py b/nebula/fixtures.py new file mode 100644 index 00000000..fbb11a26 --- /dev/null +++ b/nebula/fixtures.py @@ -0,0 +1,94 @@ +import logging +import os + +import pytest + +import nebula + +logging.getLogger().setLevel(logging.INFO) + + +class MyFilter(logging.Filter): + def filter(self, record): + return "nebula" in record.name + + +def pytest_addoption(parser): + group = parser.getgroup("nebula") + group.addoption( + "--enable-update", + action="store_true", + dest="enable_update", + default=False, + help="Update boot files and reboot board(s) with nebula", + ) + group.addoption( + "--nb-log-level", + action="store", + dest="nb_log_level", + default="ERROR", + help="Set log level for nebula", + ) + + +@pytest.fixture(scope="function") +def sd_card_update_boot(request): + """pytest fixture to update SD card and reboot board(s) with nebula""" + enable_update = request.config.getoption("--enable-update") + if not enable_update: + yield + return + + marker = request.node.get_closest_marker("nebula_update_boot") + + if not marker: + yield + return + + board_name = marker.args[0] + if not hasattr(pytest, "nebula_boards_booted"): + pytest.nebula_boards_booted = [] + if board_name not in pytest.nebula_boards_booted: + pytest.nebula_boards_booted.append(board_name) + + print("Running boot test for board: " + board_name) + + # board_name = "zynq-adrv9364" + yamlfilename = "/tmp/hw_test/test.yaml" + folder = "/tmp/hw_test/boot_files" + + level = request.config.getoption("--nb-log-level") + if level not in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]: + raise ValueError("Invalid nebula log level: " + level) + + log = logging.getLogger("nebula") + log.setLevel(getattr(logging, level)) + log = logging.getLogger() + root_handler = log.handlers[0] + root_handler.addFilter(MyFilter()) + root_handler.setFormatter( + logging.Formatter("%(levelname)s | %(name)s : %(message)s") + ) + + # Update SD card over networking + m = nebula.manager(configfilename=yamlfilename, board_name=board_name) + + m.board_reboot_auto_folder(folder, design_name=board_name) + + else: + print("Board already booted: " + board_name) + + yield + + print("Generated log files:") + + +# # Example use + +# @pytest.mark.nebula_update_boot("zynq-adrv9364") +# def test_boot_hw(sd_card_update_boot): + +# import iio +# ctx = iio.Context("ip:analog.local") +# for dev in ctx.devices: +# print(dev.name) diff --git a/nebula/helper.py b/nebula/helper.py index fe8d1bc6..137987a7 100644 --- a/nebula/helper.py +++ b/nebula/helper.py @@ -4,9 +4,10 @@ import pathlib import click -import nebula.errors as ne import netifaces import yaml + +import nebula.errors as ne from nebula.common import multi_device_check from nebula.netbox import NetboxDevice, NetboxDevices, netbox diff --git a/nebula/main.py b/nebula/main.py index e2f07868..31f7d65a 100644 --- a/nebula/main.py +++ b/nebula/main.py @@ -1,4 +1,5 @@ from invoke import Collection, Program + from nebula import tasks program = Program(namespace=Collection.from_module(tasks), version="0.0.1") diff --git a/nebula/manager.py b/nebula/manager.py index 100ecd1b..16e84bf5 100644 --- a/nebula/manager.py +++ b/nebula/manager.py @@ -4,10 +4,11 @@ import tarfile import time +import yaml + import nebula.common as common import nebula.errors as ne import nebula.helper as helper -import yaml from nebula.driver import driver from nebula.jtag import jtag from nebula.netconsole import netconsole @@ -196,7 +197,7 @@ def network_check(self): self.monitor[0].stop_log() raise ne.SSHNotFunctionalAfterBootFileUpdate - @_release_thread_lock + @_release_thread_lock # type: ignore def recover_board( self, system_top_bit_path, bootbinpath, uimagepath, devtreepath, sdcard=False ): @@ -309,7 +310,7 @@ def recover_board( bootbinpath, uimagepath, devtreepath, sdcard ) - @_release_thread_lock + @_release_thread_lock # type: ignore def board_reboot_jtag_uart( self, bootbinpath, uimagepath, devtreepath, sdcard=False ): @@ -322,7 +323,7 @@ def board_reboot_jtag_uart( # Check if u-boot loads first # log.info("Resetting with JTAG and checking if u-boot is reachable") # self.jtag.restart_board() - # do a power cylcle rather than jtag reboot to make sure jtag devices are working + # do a power cycle rather than jtag reboot to make sure jtag devices are working log.info("Resetting with JTAG and checking if u-boot is reachable") self.jtag.restart_board() if self.monitor[0]._enter_uboot_menu_from_power_cycle(): @@ -362,7 +363,7 @@ def board_reboot_jtag_uart( self.monitor[0].stop_log() - @_release_thread_lock + @_release_thread_lock # type: ignore def board_reboot_uart_net_pdu( self, system_top_bit_path, bootbinpath, uimagepath, devtreepath ): @@ -642,7 +643,9 @@ def board_reboot_auto_folder( else: log.info("SD-Card/microblaze based device selected") (bootbin, kernel, dt, bit) = self._find_boot_files(folder) - print(bootbin, kernel, dt, bit) + log.info("Found boot files:") + for file in [bootbin, kernel, dt, bit]: + log.info(file) if jtag_mode: self.board_reboot_jtag_uart( system_top_bit_path=bit, diff --git a/nebula/netbox.py b/nebula/netbox.py index 938805f9..5819041d 100644 --- a/nebula/netbox.py +++ b/nebula/netbox.py @@ -7,9 +7,10 @@ import pynetbox import yaml -from nebula.common import utils from numpy import isin +from nebula.common import utils + log = logging.getLogger(__name__) diff --git a/nebula/network.py b/nebula/network.py index 072e4f3f..c05dace5 100644 --- a/nebula/network.py +++ b/nebula/network.py @@ -8,8 +8,9 @@ import time import fabric -import nebula.errors as ne from fabric import Connection + +import nebula.errors as ne from nebula.common import utils log = logging.getLogger(__name__) @@ -85,7 +86,13 @@ def check_ssh(self): result = fabric.Connection( self.dutusername + "@" + self.dutip, connect_kwargs={"password": self.dutpassword}, - ).run("uname -a", hide=True, timeout=self.ssh_timeout) + ).run( + "uname -a", + hide=True, + timeout=self.ssh_timeout, + pty=True, + in_stream=False, + ) break except Exception as ex: log.warning("Exception raised: " + str(ex)) @@ -146,7 +153,13 @@ def run_ssh_command( result = fabric.Connection( self.dutusername + "@" + self.dutip, connect_kwargs={"password": self.dutpassword}, - ).run(command, hide=True, timeout=self.ssh_timeout) + ).run( + command, + hide=True, + timeout=self.ssh_timeout, + pty=True, + in_stream=False, + ) if result.failed: raise Exception("Failed to run command:", command) diff --git a/nebula/pdu.py b/nebula/pdu.py index 8c55c010..76196f1c 100644 --- a/nebula/pdu.py +++ b/nebula/pdu.py @@ -1,9 +1,10 @@ import logging import time +from pyvesync_v2 import VeSync + from nebula import cyberpower as cpdu from nebula.common import utils -from pyvesync_v2 import VeSync log = logging.getLogger(__name__) diff --git a/nebula/resources/template_gen.yaml b/nebula/resources/template_gen.yaml index 88ce52e0..d2d34778 100644 --- a/nebula/resources/template_gen.yaml +++ b/nebula/resources/template_gen.yaml @@ -26,7 +26,7 @@ board-config: field_3: name: daughter default: FMCOMMS2-3 - help: "Daugther board name" + help: "Daughter board name" optional: False netbox_field: devices.custom_fields.device_daughter field_4: diff --git a/nebula/resources/template_rpi_gen.yaml b/nebula/resources/template_rpi_gen.yaml index 13f81d14..87d918f7 100644 --- a/nebula/resources/template_rpi_gen.yaml +++ b/nebula/resources/template_rpi_gen.yaml @@ -25,7 +25,7 @@ board-config: netbox_field: devices.custom_fields.device_carrier field_3: name: daughter - help: "Daugther board name" + help: "Daughter board name" optional: True netbox_field: devices.custom_fields.device_daughter field_4: diff --git a/nebula/tasks.py b/nebula/tasks.py index 6e7292f5..5799ac3d 100644 --- a/nebula/tasks.py +++ b/nebula/tasks.py @@ -3,10 +3,11 @@ import time from operator import truediv -import nebula import yaml from invoke import Collection, task +import nebula + logging.getLogger().setLevel(logging.WARNING) diff --git a/nebula/uart.py b/nebula/uart.py index 8c3c6b6c..481d34c3 100644 --- a/nebula/uart.py +++ b/nebula/uart.py @@ -9,9 +9,10 @@ import serial import xmodem -from nebula.common import utils from tqdm import tqdm +from nebula.common import utils + log = logging.getLogger(__name__) LINUX_SERIAL_FOLDER = "/dev/serial" @@ -132,7 +133,6 @@ def start_log(self, logappend=False, force=False): """Trigger monitoring with UART interface""" if not self.listen_thread_run or force: self.listen_thread_run = True - print("STARTING UART LOG") log.info("Launching UART listening thread") if not self.print_to_console: log.info("UART console saving to file: " + self.logfilename) diff --git a/nebula/usbmux.py b/nebula/usbmux.py index 362991dc..6d671b1f 100644 --- a/nebula/usbmux.py +++ b/nebula/usbmux.py @@ -9,9 +9,10 @@ from pathlib import Path import pyudev -from nebula.common import utils from usbsdmux import usbsdmux +from nebula.common import utils + log = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index 56bd8510..c9ce9cfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ pyfiglet = "^0.8.post1" fabric = "^2.5.0" pyyaml = "^5.3.1" pyserial = "^3.4" -pytest = "^5.4.1" +pytest = "^7.0.0" pyvesync_v2 = "^1.1.0" pysnmp = "^4.4.12" xmodem = "^0.4.6" @@ -24,3 +24,9 @@ sphinx = "^3.0.1" [build-system] requires = ["poetry>=0.12"] build-backend = "poetry.masonry.api" + +[tool.poetry.plugins.pytest11] +nebula = "nebula.fixtures" + +[tool.poetry.scripts] +nebula = "nebula.main:program.run" diff --git a/setup.cfg b/setup.cfg index 43aa3910..c0f51c48 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,3 +14,4 @@ exclude = test/*,examples/*,doc/*,images/* [mypy] ignore_missing_imports=true +disable_error_code=import-untyped diff --git a/setup.py b/setup.py index e7293c3b..db4514f3 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ def _get_version(): "pysnmp", "invoke", "xmodem", - "pytest", + "pytest>=7.0.0", "pyvesync_v2", "pyfiglet", "usbsdmux", diff --git a/tests/conftest.py b/tests/conftest.py index 462497b5..df40289f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import pytest + from nebula import pdu diff --git a/tests/test_builders.py b/tests/test_builders.py index beb7330b..22591f66 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -2,6 +2,7 @@ import shutil import pytest + from nebula import builder diff --git a/tests/test_cli.py b/tests/test_cli.py index 641bd121..86a44613 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -6,6 +6,7 @@ import pytest from fabric import Connection as con + from nebula import pdu, uart # must include -s diff --git a/tests/test_downloader.py b/tests/test_downloader.py index e212765e..29c18420 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -2,6 +2,7 @@ import shutil import pytest + from nebula import downloader # Must be connected to analog VPN @@ -86,7 +87,7 @@ def test_noos_downloader(test_downloader, board_name, branch, filetype): assert os.path.isfile("outs/properties.yaml") -@pytest.mark.parametrize("board_name", ["kc705_fmcomms4"]) +@pytest.mark.parametrize("board_name", ["vcu118_ad9081_fmca_ebz"]) @pytest.mark.parametrize("branch", ["release", "master"]) @pytest.mark.parametrize("filetype", ["microblaze"]) def test_microblaze_downloader(test_downloader, board_name, branch, filetype): diff --git a/tests/test_driver.py b/tests/test_driver.py index b3049b1f..b1e663d2 100644 --- a/tests/test_driver.py +++ b/tests/test_driver.py @@ -2,6 +2,7 @@ import shutil import pytest + from nebula import driver # def remove_file(files): diff --git a/tests/test_manager.py b/tests/test_manager.py index e6ba0f6d..2e50e7f9 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -3,6 +3,7 @@ import time import pytest + from nebula import manager diff --git a/tests/test_network.py b/tests/test_network.py index efdf7f7a..a5a44786 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -3,6 +3,7 @@ import time import pytest + from nebula import network, pdu, uart diff --git a/tests/test_uart.py b/tests/test_uart.py index 815c2a7d..6fca532e 100644 --- a/tests/test_uart.py +++ b/tests/test_uart.py @@ -3,6 +3,7 @@ import time import pytest + from nebula import pdu, uart diff --git a/tests/test_usbsdmux.py b/tests/test_usbsdmux.py index 3f8ae7c8..64a99804 100644 --- a/tests/test_usbsdmux.py +++ b/tests/test_usbsdmux.py @@ -2,6 +2,7 @@ import shutil import pytest + from nebula import pdu, usbmux