From 9147ade6a90f28d69c07f1a5884c5cf5e337effe Mon Sep 17 00:00:00 2001 From: Christian Heider Nielsen Date: Fri, 16 Sep 2022 09:29:53 +0200 Subject: [PATCH] pin pypi release action --- .github/workflows/publish-to-test-pypi.yml | 4 +- docs/source/conf.py | 20 +--- heimdallr/configuration/heimdallr_settings.py | 34 +++--- heimdallr/entry_points/cli.py | 2 +- .../deprecated/publisher_non_sch.py | 16 +-- heimdallr/entry_points/publisher.py | 24 +++-- heimdallr/entry_points/server.py | 12 ++- heimdallr/server/__init__.py | 8 ++ .../utilities/publisher/system_resources.py | 100 ++++++++---------- .../windows_task_scheduler_utilities.py | 2 - heimdallr/utilities/server/google_calendar.py | 2 +- requirements/requirements_server.txt | 2 +- samples/pynvml_example.py | 13 ++- tests/test_nvml.py | 12 +-- 14 files changed, 127 insertions(+), 124 deletions(-) diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index f64598f..b727b06 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -34,7 +34,7 @@ jobs: env: test_pypi_password: ${{ secrets.test_pypi_secret }} if: env.test_pypi_password != null && endsWith(github.ref, 'master') && github.repository_owner == 'aivclab' - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.test_pypi_secret }} repository_url: https://test.pypi.org/legacy/ @@ -42,7 +42,7 @@ jobs: env: pypi_password: ${{ secrets.pypi_secret }} if: env.pypi_password != null && startsWith(github.ref, 'refs/tags') && github.repository_owner == 'aivclab' - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_secret }} #verbose: true diff --git a/docs/source/conf.py b/docs/source/conf.py index 238e649..39ee52d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,22 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# -# Neo documentation build configuration file, created by -# sphinx-quickstart on Tue Jul 25 10:23:12 2017. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# import sys from pathlib import Path @@ -110,7 +94,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] +exclude_patterns = ['**/*exclude*', '**/*Exclude*'] # The name of the Pygments (syntax highlighting) style to use. highlight_language = "python" @@ -142,7 +126,7 @@ # "display_version": False, } -html_baseurl = f"aivclab.github.io/{PROJECT_NAME}" +html_baseurl = f"{PROJECT_ORGANISATION}.github.io/{PROJECT_NAME}" # -- Options for HTMLHelp output ------------------------------------------ diff --git a/heimdallr/configuration/heimdallr_settings.py b/heimdallr/configuration/heimdallr_settings.py index 119b74e..3b5a077 100644 --- a/heimdallr/configuration/heimdallr_settings.py +++ b/heimdallr/configuration/heimdallr_settings.py @@ -18,7 +18,7 @@ from sorcery import assigned_names -from apppath import ensure_existence +from warg import ensure_existence from heimdallr import PROJECT_APP_PATH from warg import PropertySettings, is_windows @@ -182,14 +182,11 @@ def google_calendar_id(self, calendar_id: str) -> None: @google_calendar_id.deleter def google_calendar_id(self) -> None: - """ - - Args: - calendar_id (): + ''' - Returns: - - """ + :return: + :rtype: + ''' key = inspect.currentframe().f_code.co_name with shelve.open( str(HeimdallrSettings._github_settings_path), writeback=True @@ -233,14 +230,11 @@ def github_token(self, calendar_id: str) -> None: @github_token.deleter def github_token(self) -> None: - """ - - Args: - calendar_id (): + ''' - Returns: - - """ + :return: + :rtype: + ''' key = inspect.currentframe().f_code.co_name with shelve.open( str(HeimdallrSettings._github_settings_path), writeback=True @@ -468,18 +462,18 @@ def set_all_heimdallr_settings( **kwargs, ): """description""" - HEIMDALLR_SETTINGS = HeimdallrSettings(setting_scope) - # print(f"current heimdallr settings: {HEIMDALLR_SETTINGS}") + heimdallr_settings = HeimdallrSettings(setting_scope) + # print(f"current heimdallr settings: {heimdallr_settings}") if _lower_keys: kwargs = {k.lower(): v for k, v in kwargs.items()} # for k in kwargs.keys(): - # assert k in HEIMDALLR_SETTINGS, f'"{k}" is not in Heimdallrs settings' + # assert k in heimdallr_settings, f'"{k}" is not in Heimdallrs settings' - for k in HEIMDALLR_SETTINGS: + for k in heimdallr_settings: assert k in kwargs.keys(), f'Missing "{k}" from kwargs' - HEIMDALLR_SETTINGS.__from_dict__(kwargs) + heimdallr_settings.__from_dict__(kwargs) print(f"new heimdallr settings: {HeimdallrSettings()}") diff --git a/heimdallr/entry_points/cli.py b/heimdallr/entry_points/cli.py index 6ae375c..94e2892 100644 --- a/heimdallr/entry_points/cli.py +++ b/heimdallr/entry_points/cli.py @@ -90,7 +90,7 @@ def set( """Setting options: [mqtt_access_token, mqtt_username, mqtt_password, mqtt_broker, mqtt_port]""" print(self.setting_scope) settings = HeimdallrSettings(setting_scope=self.setting_scope) - print(settings._mqtt_settings_path) + print(settings._mqtt_settings_path) #TODO: ACCESS TO protected member settings.__setattr__(setting, value) def multi_set(self, **kw) -> None: diff --git a/heimdallr/entry_points/deprecated/publisher_non_sch.py b/heimdallr/entry_points/deprecated/publisher_non_sch.py index f270152..8454ff0 100644 --- a/heimdallr/entry_points/deprecated/publisher_non_sch.py +++ b/heimdallr/entry_points/deprecated/publisher_non_sch.py @@ -3,8 +3,8 @@ import time import paho.mqtt.client as mqtt -from apppath import ensure_existence -from draugr import IgnoreInterruptSignal +from warg import ensure_existence +from warg import IgnoreInterruptSignal from warg import busy_indicator from draugr.writers import LogWriter, MockWriter, Writer from warg import NOD @@ -52,20 +52,20 @@ def main(is_user: bool = False): client.on_publish = on_publish client.on_disconnect = on_disconnect - HEIMDALLR_SETTINGS = HeimdallrSettings() # TODO input scope + heimdallr_settings = HeimdallrSettings() # TODO input scope client.username_pw_set( - HEIMDALLR_SETTINGS.mqtt_username, HEIMDALLR_SETTINGS.mqtt_password + heimdallr_settings.mqtt_username, heimdallr_settings.mqtt_password ) try: client.connect( - HEIMDALLR_SETTINGS.mqtt_broker, HEIMDALLR_SETTINGS.mqtt_port, keepalive=60 + heimdallr_settings.mqtt_broker, heimdallr_settings.mqtt_port, keepalive=60 ) except ValueError as ve: raise ValueError( - f"{HEIMDALLR_SETTINGS._mqtt_settings_path}," - f"{HEIMDALLR_SETTINGS.mqtt_broker}," - f"{HEIMDALLR_SETTINGS.mqtt_port}," + f"{heimdallr_settings._mqtt_settings_path}," + f"{heimdallr_settings.mqtt_broker}," + f"{heimdallr_settings.mqtt_port}," f"{ve}" ) client.loop_start() diff --git a/heimdallr/entry_points/publisher.py b/heimdallr/entry_points/publisher.py index e938c3b..c4b2ce0 100644 --- a/heimdallr/entry_points/publisher.py +++ b/heimdallr/entry_points/publisher.py @@ -1,3 +1,13 @@ + +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +__author__ = "Christian Heider Nielsen" +__doc__ = r""" + + Created on 19/03/2020 + """ + import json import socket import time @@ -5,7 +15,7 @@ import paho.mqtt.client as mqtt import schedule -from apppath import ensure_existence +from warg import ensure_existence from warg import busy_indicator from draugr.writers import LogWriter, MockWriter, Writer from warg import NOD @@ -66,20 +76,20 @@ def main(setting_scope: SettingScopeEnum = SettingScopeEnum.user) -> None: client.on_publish = on_publish client.on_disconnect = on_disconnect - HEIMDALLR_SETTINGS = HeimdallrSettings(setting_scope) + heimdallr_settings = HeimdallrSettings(setting_scope) client.username_pw_set( - HEIMDALLR_SETTINGS.mqtt_username, HEIMDALLR_SETTINGS.mqtt_password + heimdallr_settings.mqtt_username, heimdallr_settings.mqtt_password ) try: client.connect( - HEIMDALLR_SETTINGS.mqtt_broker, HEIMDALLR_SETTINGS.mqtt_port, keepalive=60 + heimdallr_settings.mqtt_broker, heimdallr_settings.mqtt_port, keepalive=60 ) except ValueError as ve: raise ValueError( - f"{HEIMDALLR_SETTINGS._mqtt_settings_path}," - f"{HEIMDALLR_SETTINGS.mqtt_broker}," - f"{HEIMDALLR_SETTINGS.mqtt_port}," + f"{heimdallr_settings._mqtt_settings_path}," + f"{heimdallr_settings.mqtt_broker}," + f"{heimdallr_settings.mqtt_port}," f"{ve}" ) diff --git a/heimdallr/entry_points/server.py b/heimdallr/entry_points/server.py index ca77ea7..b779fff 100644 --- a/heimdallr/entry_points/server.py +++ b/heimdallr/entry_points/server.py @@ -1,3 +1,13 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +__author__ = "Christian Heider Nielsen" +__doc__ = r""" + + Created on 19/03/2020 + """ + + import copy import datetime import json @@ -7,7 +17,7 @@ import dash import flask -from apppath import ensure_existence +from warg import ensure_existence from dash import Dash from dash.dash_table import DataTable from dash.dependencies import Input, Output diff --git a/heimdallr/server/__init__.py b/heimdallr/server/__init__.py index e69de29..5c1bcaf 100644 --- a/heimdallr/server/__init__.py +++ b/heimdallr/server/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +__author__ = "Christian Heider Nielsen" +__doc__ = r""" + + Created on 19/03/2020 + """ diff --git a/heimdallr/utilities/publisher/system_resources.py b/heimdallr/utilities/publisher/system_resources.py index 959163b..dfac94f 100644 --- a/heimdallr/utilities/publisher/system_resources.py +++ b/heimdallr/utilities/publisher/system_resources.py @@ -7,72 +7,62 @@ Created on 29/03/2020 """ -from typing import Dict, Mapping, Tuple +from typing import Dict, Tuple import psutil __all__ = ["get_list_of_process_sorted_by_memory"] - -def select(mapping: Mapping, *a) -> Mapping: - """ - Select keys from mapping if in a - - Args: - mapping: - *a: - - Returns: - - """ - return {k: v for k, v in mapping.items() if k in a} +from warg.generators.mapping_generator import select_dict def get_list_of_process_sorted_by_memory( - attrs: Tuple = ("name", "username"), scaling: float = (1024**2), top_k: int = 10 -) -> Dict: - """ - Get list of running process sorted by Memory Usage - """ - list_of_proc_objects = [] # TODO: REFACTOR TO PID,VAL DICT DIRECTLY? - for proc in psutil.process_iter(): - try: - proc_info = proc.as_dict(attrs={"pid", *attrs}) - proc_info["vms"] = proc.memory_info().vms / scaling - list_of_proc_objects.append(proc_info) - except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): - pass - sorted_entries = sorted( - list_of_proc_objects, key=lambda proc_obj: proc_obj["vms"], reverse=True - ) - if top_k: - sorted_entries = sorted_entries[:top_k] - return {v["pid"]: select(v, "vms", *attrs) for v in sorted_entries} + attrs: Tuple = ("name", "username"), scaling: float = (1024 ** 2), top_k: int = 10 + ) -> Dict: + """ + Get list of running process sorted by Memory Usage + """ + list_of_proc_objects = [] # TODO: REFACTOR TO PID,VAL DICT DIRECTLY? + for proc in psutil.process_iter(): + try: + proc_info = proc.as_dict(attrs = {"pid", *attrs}) + proc_info["vms"] = proc.memory_info().vms / scaling + list_of_proc_objects.append(proc_info) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + pass + sorted_entries = sorted( + list_of_proc_objects, key = lambda proc_obj: proc_obj["vms"], reverse = True + ) + if top_k: + sorted_entries = sorted_entries[:top_k] + return {v["pid"]: select_dict(v, "vms", *attrs) for v in sorted_entries} if __name__ == "__main__": - def main(): - """description""" - print("*** Iterate over all running process and print process ID & Name ***") - for proc in psutil.process_iter(): - try: - process_name = proc.name() - process_id = proc.pid - print(process_name, " ::: ", process_id) - except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): - pass - print("*** Create a list of all running processes ***") + def main(): + """description""" + print("*** Iterate over all running process and print process ID & Name ***") + for proc in psutil.process_iter(): + try: + process_name = proc.name() + process_id = proc.pid + print(process_name, " ::: ", process_id) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + pass + print("*** Create a list of all running processes ***") + + + def all_info_procs(): + """description""" + list_of_process_names = list() + for proc in psutil.process_iter(): + p_info_dict = proc.as_dict() + list_of_process_names.append(p_info_dict) + for elem in list_of_process_names: + print(elem) - def all_info_procs(): - """description""" - list_of_process_names = list() - for proc in psutil.process_iter(): - p_info_dict = proc.as_dict() - list_of_process_names.append(p_info_dict) - for elem in list_of_process_names: - print(elem) - # main() - print(get_list_of_process_sorted_by_memory()) - # all_info_procs() + # main() + print(get_list_of_process_sorted_by_memory()) + # all_info_procs() diff --git a/heimdallr/utilities/publisher/windows_task_scheduler_utilities.py b/heimdallr/utilities/publisher/windows_task_scheduler_utilities.py index 6e4ed03..fded6df 100644 --- a/heimdallr/utilities/publisher/windows_task_scheduler_utilities.py +++ b/heimdallr/utilities/publisher/windows_task_scheduler_utilities.py @@ -9,8 +9,6 @@ __all__ = ["disable_service", "remove_service", "install_service", "enable_service"] -from pathlib import Path - from draugr.os_utilities.windows_utilities import ( delete_task, new_user_logon_execute_task, diff --git a/heimdallr/utilities/server/google_calendar.py b/heimdallr/utilities/server/google_calendar.py index a889c60..c733069 100644 --- a/heimdallr/utilities/server/google_calendar.py +++ b/heimdallr/utilities/server/google_calendar.py @@ -20,7 +20,7 @@ except: print("try upgrade google-api-python-client") -from apppath import ensure_existence +from warg import ensure_existence from heimdallr.configuration.heimdallr_settings import HeimdallrSettings from heimdallr.utilities.date_tools import iso_dt_to_datetime diff --git a/requirements/requirements_server.txt b/requirements/requirements_server.txt index 99e204b..ae8363c 100644 --- a/requirements/requirements_server.txt +++ b/requirements/requirements_server.txt @@ -7,7 +7,7 @@ pandas scikit-learn torch psutil -dash +dash>=2.6.1 #dash_bootstrap_components plotly google_auth_oauthlib diff --git a/samples/pynvml_example.py b/samples/pynvml_example.py index 0575c32..97aa80c 100644 --- a/samples/pynvml_example.py +++ b/samples/pynvml_example.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +__author__ = "Christian Heider Nielsen" +__doc__ = r""" + + Created on 02-12-2020 + """ + ################################################################################# # Copyright (c) 2020, NVIDIA Corporation. All rights reserved. # # # @@ -138,9 +147,9 @@ def device_query(): for i in range(0, device_count): handle = nvmlDeviceGetHandleByIndex(i) - pciInfo = nvmlDeviceGetPciInfo(handle) + pci_nfo = nvmlDeviceGetPciInfo(handle) - str_result += f' \n' + str_result += f' \n' str_result += ( f" {str(nvmlDeviceGetName(handle))}\n" diff --git a/tests/test_nvml.py b/tests/test_nvml.py index b8c6623..fd9e80a 100644 --- a/tests/test_nvml.py +++ b/tests/test_nvml.py @@ -121,7 +121,7 @@ def pci_info(ngpus, handles): # Test pynvml.nvmlSystemGetNVMLVersion @pytest.mark.skipif(sys.platform != "linux", reason="Test only on linux") - def test_nvmlSystemGetNVMLVersion(nvml): + def test_nvml_system_get_nvml_version(nvml): vsn = 0.0 vsn = pynvml.nvmlSystemGetNVMLVersion().decode() print("[NVML Version: " + vsn + "]", end=" ") @@ -129,7 +129,7 @@ def test_nvmlSystemGetNVMLVersion(nvml): # Test pynvml.nvmlSystemGetProcessName @pytest.mark.skipif(sys.platform != "linux", reason="Test only on linux") - def test_nvmlSystemGetProcessName(nvml): + def test_nvml_system_get_process_name(nvml): procname = None procname = pynvml.nvmlSystemGetProcessName(os.getpid()) print("[Process: " + str(procname.decode()) + "]", end=" ") @@ -137,7 +137,7 @@ def test_nvmlSystemGetProcessName(nvml): # Test pynvml.nvmlSystemGetDriverVersion @pytest.mark.skipif(sys.platform != "linux", reason="Test only on linux") - def test_nvmlSystemGetDriverVersion(nvml): + def test_nvml_system_get_driver_version(nvml): vsn = 0.0 vsn = pynvml.nvmlSystemGetDriverVersion().decode() print("[Driver Version: " + vsn + "]", end=" ") @@ -149,19 +149,19 @@ def test_nvmlSystemGetDriverVersion(nvml): # Test pynvml.nvmlDeviceGetHandleBySerial @pytest.mark.skipif(sys.platform != "linux", reason="Test only on linux") - def test_nvmlDeviceGetHandleBySerial(ngpus, serials): + def test_nvml_device_get_handle_by_serial(ngpus, serials): handles = [pynvml.nvmlDeviceGetHandleBySerial(serials[i]) for i in range(ngpus)] assert len(handles) == ngpus # Test pynvml.nvmlDeviceGetHandleByUUID @pytest.mark.skipif(sys.platform != "linux", reason="Test only on linux") - def test_nvmlDeviceGetHandleByUUID(ngpus, uuids): + def test_nvml_device_get_handle_by_uuid(ngpus, uuids): handles = [pynvml.nvmlDeviceGetHandleByUUID(uuids[i]) for i in range(ngpus)] assert len(handles) == ngpus # Test pynvml.nvmlDeviceGetHandleByPciBusId @pytest.mark.skipif(sys.platform != "linux", reason="Test only on linux") - def test_nvmlDeviceGetHandleByPciBusId(ngpus, pci_info): + def test_nvml_device_get_handle_by_pci_bus_id(ngpus, pci_info): handles = [ pynvml.nvmlDeviceGetHandleByPciBusId(pci_info[i].busId) for i in range(ngpus)