From 5300eef27ef637ae34c410f8cc566b5a61e69319 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 27 Aug 2020 10:43:13 -0700 Subject: [PATCH 01/36] daemon: added a more specific error to be thrown when a service does not exist --- daemon/core/errors.py | 8 ++++++++ daemon/core/services/coreservices.py | 7 +++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/daemon/core/errors.py b/daemon/core/errors.py index d299f5ae7..4e6ceb929 100644 --- a/daemon/core/errors.py +++ b/daemon/core/errors.py @@ -30,3 +30,11 @@ class CoreXmlError(Exception): """ pass + + +class CoreServiceError(Exception): + """ + Used when there is an error related to accessing a service. + """ + + pass diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index 646a433dc..590f79503 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -25,7 +25,7 @@ from core import utils from core.emulator.data import FileData from core.emulator.enumerations import ExceptionLevels, MessageFlags, RegisterTlvs -from core.errors import CoreCommandError, CoreError +from core.errors import CoreCommandError, CoreError, CoreServiceError from core.nodes.base import CoreNode if TYPE_CHECKING: @@ -257,7 +257,10 @@ def get(cls, name: str) -> Type["CoreService"]: :param name: name of the service to retrieve :return: service if it exists, None otherwise """ - return cls.services.get(name) + service = cls.services.get(name) + if service is None: + raise CoreServiceError(f"service({name}) does not exist") + return service @classmethod def add_services(cls, path: str) -> List[str]: From f6992e754524c65df4b8cc66e1e999e7927c9914 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 27 Aug 2020 10:46:55 -0700 Subject: [PATCH 02/36] daemon: moved service boot error to core.errors with all other core specific errors --- daemon/core/errors.py | 8 ++++++++ daemon/core/services/coreservices.py | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/daemon/core/errors.py b/daemon/core/errors.py index 4e6ceb929..a75bd536f 100644 --- a/daemon/core/errors.py +++ b/daemon/core/errors.py @@ -38,3 +38,11 @@ class CoreServiceError(Exception): """ pass + + +class CoreServiceBootError(Exception): + """ + Used when there is an error booting a service. + """ + + pass diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index 590f79503..b4c33990a 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -25,7 +25,12 @@ from core import utils from core.emulator.data import FileData from core.emulator.enumerations import ExceptionLevels, MessageFlags, RegisterTlvs -from core.errors import CoreCommandError, CoreError, CoreServiceError +from core.errors import ( + CoreCommandError, + CoreError, + CoreServiceBootError, + CoreServiceError, +) from core.nodes.base import CoreNode if TYPE_CHECKING: @@ -34,10 +39,6 @@ CoreServiceType = Union["CoreService", Type["CoreService"]] -class ServiceBootError(Exception): - pass - - class ServiceMode(enum.Enum): BLOCKING = 0 NON_BLOCKING = 1 @@ -453,7 +454,7 @@ def boot_services(self, node: CoreNode) -> None: funcs.append((self._boot_service_path, args, {})) result, exceptions = utils.threadpool(funcs) if exceptions: - raise ServiceBootError(*exceptions) + raise CoreServiceBootError(*exceptions) def _boot_service_path(self, node: CoreNode, boot_path: List["CoreServiceType"]): logging.info( @@ -467,7 +468,7 @@ def _boot_service_path(self, node: CoreNode, boot_path: List["CoreServiceType"]) self.boot_service(node, service) except Exception as e: logging.exception("exception booting service: %s", service.name) - raise ServiceBootError(e) + raise CoreServiceBootError(e) def boot_service(self, node: CoreNode, service: "CoreServiceType") -> None: """ @@ -504,7 +505,7 @@ def boot_service(self, node: CoreNode, service: "CoreServiceType") -> None: wait = service.validation_mode == ServiceMode.BLOCKING status = self.startup_service(node, service, wait) if status: - raise ServiceBootError( + raise CoreServiceBootError( "node(%s) service(%s) error during startup" % (node.name, service.name) ) @@ -529,7 +530,7 @@ def boot_service(self, node: CoreNode, service: "CoreServiceType") -> None: time.sleep(service.validation_period) if status: - raise ServiceBootError( + raise CoreServiceBootError( "node(%s) service(%s) failed validation" % (node.name, service.name) ) From b0bac1d319ad6aaaf81057fe1f72dd52bc71b666 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Thu, 27 Aug 2020 11:02:02 -0700 Subject: [PATCH 03/36] daemon: moved grpc wrapper classes to core.grpc.wrappers --- daemon/core/{gui => api/grpc}/wrappers.py | 0 daemon/core/gui/coreclient.py | 26 +++++++++---------- daemon/core/gui/dialogs/alerts.py | 2 +- .../core/gui/dialogs/configserviceconfig.py | 8 +++--- daemon/core/gui/dialogs/emaneconfig.py | 2 +- daemon/core/gui/dialogs/hooks.py | 2 +- daemon/core/gui/dialogs/linkconfig.py | 2 +- daemon/core/gui/dialogs/mobilityconfig.py | 2 +- daemon/core/gui/dialogs/mobilityplayer.py | 2 +- daemon/core/gui/dialogs/nodeconfig.py | 2 +- daemon/core/gui/dialogs/nodeconfigservice.py | 2 +- daemon/core/gui/dialogs/nodeservice.py | 2 +- daemon/core/gui/dialogs/serviceconfig.py | 2 +- daemon/core/gui/dialogs/sessionoptions.py | 2 +- daemon/core/gui/dialogs/sessions.py | 2 +- daemon/core/gui/dialogs/wlanconfig.py | 2 +- daemon/core/gui/frames/link.py | 2 +- daemon/core/gui/frames/node.py | 2 +- daemon/core/gui/graph/edges.py | 2 +- daemon/core/gui/graph/graph.py | 9 ++++++- daemon/core/gui/graph/node.py | 2 +- daemon/core/gui/images.py | 2 +- daemon/core/gui/interface.py | 2 +- daemon/core/gui/nodeutils.py | 2 +- daemon/core/gui/statusbar.py | 2 +- daemon/core/gui/widgets.py | 2 +- 26 files changed, 47 insertions(+), 40 deletions(-) rename daemon/core/{gui => api/grpc}/wrappers.py (100%) diff --git a/daemon/core/gui/wrappers.py b/daemon/core/api/grpc/wrappers.py similarity index 100% rename from daemon/core/gui/wrappers.py rename to daemon/core/api/grpc/wrappers.py diff --git a/daemon/core/gui/coreclient.py b/daemon/core/gui/coreclient.py index 902f780af..116b1617b 100644 --- a/daemon/core/gui/coreclient.py +++ b/daemon/core/gui/coreclient.py @@ -21,19 +21,7 @@ services_pb2, wlan_pb2, ) -from core.gui import appconfig -from core.gui.appconfig import BACKGROUNDS_PATH, XMLS_PATH, CoreServer, Observer -from core.gui.dialogs.emaneinstall import EmaneInstallDialog -from core.gui.dialogs.error import ErrorDialog -from core.gui.dialogs.mobilityplayer import MobilityPlayer -from core.gui.dialogs.sessions import SessionsDialog -from core.gui.graph.edges import CanvasEdge -from core.gui.graph.node import CanvasNode -from core.gui.graph.shape import AnnotationData, Shape -from core.gui.graph.shapeutils import ShapeType -from core.gui.interface import InterfaceManager -from core.gui.nodeutils import NodeDraw, NodeUtils -from core.gui.wrappers import ( +from core.api.grpc.wrappers import ( ConfigOption, ConfigService, ExceptionEvent, @@ -52,6 +40,18 @@ SessionState, ThroughputsEvent, ) +from core.gui import appconfig +from core.gui.appconfig import BACKGROUNDS_PATH, XMLS_PATH, CoreServer, Observer +from core.gui.dialogs.emaneinstall import EmaneInstallDialog +from core.gui.dialogs.error import ErrorDialog +from core.gui.dialogs.mobilityplayer import MobilityPlayer +from core.gui.dialogs.sessions import SessionsDialog +from core.gui.graph.edges import CanvasEdge +from core.gui.graph.node import CanvasNode +from core.gui.graph.shape import AnnotationData, Shape +from core.gui.graph.shapeutils import ShapeType +from core.gui.interface import InterfaceManager +from core.gui.nodeutils import NodeDraw, NodeUtils if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/alerts.py b/daemon/core/gui/dialogs/alerts.py index a01937274..9e4302141 100644 --- a/daemon/core/gui/dialogs/alerts.py +++ b/daemon/core/gui/dialogs/alerts.py @@ -5,10 +5,10 @@ from tkinter import ttk from typing import TYPE_CHECKING, Dict, Optional +from core.api.grpc.wrappers import ExceptionEvent, ExceptionLevel from core.gui.dialogs.dialog import Dialog from core.gui.themes import PADX, PADY from core.gui.widgets import CodeText -from core.gui.wrappers import ExceptionEvent, ExceptionLevel if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/configserviceconfig.py b/daemon/core/gui/dialogs/configserviceconfig.py index a085afd1f..14388f5a0 100644 --- a/daemon/core/gui/dialogs/configserviceconfig.py +++ b/daemon/core/gui/dialogs/configserviceconfig.py @@ -8,15 +8,15 @@ import grpc -from core.gui.dialogs.dialog import Dialog -from core.gui.themes import FRAME_PAD, PADX, PADY -from core.gui.widgets import CodeText, ConfigFrame, ListboxScroll -from core.gui.wrappers import ( +from core.api.grpc.wrappers import ( ConfigOption, ConfigServiceData, Node, ServiceValidationMode, ) +from core.gui.dialogs.dialog import Dialog +from core.gui.themes import FRAME_PAD, PADX, PADY +from core.gui.widgets import CodeText, ConfigFrame, ListboxScroll if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/emaneconfig.py b/daemon/core/gui/dialogs/emaneconfig.py index d47a3c0d1..0829907a6 100644 --- a/daemon/core/gui/dialogs/emaneconfig.py +++ b/daemon/core/gui/dialogs/emaneconfig.py @@ -8,11 +8,11 @@ import grpc +from core.api.grpc.wrappers import ConfigOption, Node from core.gui.dialogs.dialog import Dialog from core.gui.images import ImageEnum, Images from core.gui.themes import PADX, PADY from core.gui.widgets import ConfigFrame -from core.gui.wrappers import ConfigOption, Node if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/hooks.py b/daemon/core/gui/dialogs/hooks.py index e831b4f92..474dc2d04 100644 --- a/daemon/core/gui/dialogs/hooks.py +++ b/daemon/core/gui/dialogs/hooks.py @@ -2,10 +2,10 @@ from tkinter import ttk from typing import TYPE_CHECKING, Optional +from core.api.grpc.wrappers import Hook, SessionState from core.gui.dialogs.dialog import Dialog from core.gui.themes import PADX, PADY from core.gui.widgets import CodeText, ListboxScroll -from core.gui.wrappers import Hook, SessionState if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/linkconfig.py b/daemon/core/gui/dialogs/linkconfig.py index a09cfe7fb..82bf92e19 100644 --- a/daemon/core/gui/dialogs/linkconfig.py +++ b/daemon/core/gui/dialogs/linkconfig.py @@ -5,11 +5,11 @@ from tkinter import ttk from typing import TYPE_CHECKING, Optional +from core.api.grpc.wrappers import Interface, Link, LinkOptions from core.gui import validation from core.gui.dialogs.colorpicker import ColorPickerDialog from core.gui.dialogs.dialog import Dialog from core.gui.themes import PADX, PADY -from core.gui.wrappers import Interface, Link, LinkOptions if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/mobilityconfig.py b/daemon/core/gui/dialogs/mobilityconfig.py index 80c3ca229..b22c5feff 100644 --- a/daemon/core/gui/dialogs/mobilityconfig.py +++ b/daemon/core/gui/dialogs/mobilityconfig.py @@ -7,10 +7,10 @@ import grpc +from core.api.grpc.wrappers import ConfigOption, Node from core.gui.dialogs.dialog import Dialog from core.gui.themes import PADX, PADY from core.gui.widgets import ConfigFrame -from core.gui.wrappers import ConfigOption, Node if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/mobilityplayer.py b/daemon/core/gui/dialogs/mobilityplayer.py index 352a37393..f27a36353 100644 --- a/daemon/core/gui/dialogs/mobilityplayer.py +++ b/daemon/core/gui/dialogs/mobilityplayer.py @@ -4,10 +4,10 @@ import grpc +from core.api.grpc.wrappers import MobilityAction, Node from core.gui.dialogs.dialog import Dialog from core.gui.images import ImageEnum from core.gui.themes import PADX, PADY -from core.gui.wrappers import MobilityAction, Node if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/nodeconfig.py b/daemon/core/gui/dialogs/nodeconfig.py index d81032837..6327206fa 100644 --- a/daemon/core/gui/dialogs/nodeconfig.py +++ b/daemon/core/gui/dialogs/nodeconfig.py @@ -7,6 +7,7 @@ import netaddr from PIL.ImageTk import PhotoImage +from core.api.grpc.wrappers import Node from core.gui import nodeutils, validation from core.gui.appconfig import ICONS_PATH from core.gui.dialogs.dialog import Dialog @@ -15,7 +16,6 @@ from core.gui.nodeutils import NodeUtils from core.gui.themes import FRAME_PAD, PADX, PADY from core.gui.widgets import ListboxScroll, image_chooser -from core.gui.wrappers import Node if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/nodeconfigservice.py b/daemon/core/gui/dialogs/nodeconfigservice.py index fefdc4c5e..2141b3dc5 100644 --- a/daemon/core/gui/dialogs/nodeconfigservice.py +++ b/daemon/core/gui/dialogs/nodeconfigservice.py @@ -6,11 +6,11 @@ from tkinter import messagebox, ttk from typing import TYPE_CHECKING, Optional, Set +from core.api.grpc.wrappers import Node from core.gui.dialogs.configserviceconfig import ConfigServiceConfigDialog from core.gui.dialogs.dialog import Dialog from core.gui.themes import FRAME_PAD, PADX, PADY from core.gui.widgets import CheckboxList, ListboxScroll -from core.gui.wrappers import Node if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/nodeservice.py b/daemon/core/gui/dialogs/nodeservice.py index a35e1d537..09732e73b 100644 --- a/daemon/core/gui/dialogs/nodeservice.py +++ b/daemon/core/gui/dialogs/nodeservice.py @@ -5,11 +5,11 @@ from tkinter import messagebox, ttk from typing import TYPE_CHECKING, Optional, Set +from core.api.grpc.wrappers import Node from core.gui.dialogs.dialog import Dialog from core.gui.dialogs.serviceconfig import ServiceConfigDialog from core.gui.themes import FRAME_PAD, PADX, PADY from core.gui.widgets import CheckboxList, ListboxScroll -from core.gui.wrappers import Node if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/serviceconfig.py b/daemon/core/gui/dialogs/serviceconfig.py index a22b1afd4..541a490eb 100644 --- a/daemon/core/gui/dialogs/serviceconfig.py +++ b/daemon/core/gui/dialogs/serviceconfig.py @@ -7,12 +7,12 @@ import grpc from PIL.ImageTk import PhotoImage +from core.api.grpc.wrappers import Node, NodeServiceData, ServiceValidationMode from core.gui.dialogs.copyserviceconfig import CopyServiceConfigDialog from core.gui.dialogs.dialog import Dialog from core.gui.images import ImageEnum, Images from core.gui.themes import FRAME_PAD, PADX, PADY from core.gui.widgets import CodeText, ListboxScroll -from core.gui.wrappers import Node, NodeServiceData, ServiceValidationMode if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/sessionoptions.py b/daemon/core/gui/dialogs/sessionoptions.py index e9b032e0c..4b086d670 100644 --- a/daemon/core/gui/dialogs/sessionoptions.py +++ b/daemon/core/gui/dialogs/sessionoptions.py @@ -5,10 +5,10 @@ import grpc +from core.api.grpc.wrappers import ConfigOption from core.gui.dialogs.dialog import Dialog from core.gui.themes import PADX, PADY from core.gui.widgets import ConfigFrame -from core.gui.wrappers import ConfigOption if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/sessions.py b/daemon/core/gui/dialogs/sessions.py index 4c9ae0cac..71a33fd65 100644 --- a/daemon/core/gui/dialogs/sessions.py +++ b/daemon/core/gui/dialogs/sessions.py @@ -5,11 +5,11 @@ import grpc +from core.api.grpc.wrappers import SessionState, SessionSummary from core.gui.dialogs.dialog import Dialog from core.gui.images import ImageEnum, Images from core.gui.task import ProgressTask from core.gui.themes import PADX, PADY -from core.gui.wrappers import SessionState, SessionSummary if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/dialogs/wlanconfig.py b/daemon/core/gui/dialogs/wlanconfig.py index 283a96cda..05362cc6a 100644 --- a/daemon/core/gui/dialogs/wlanconfig.py +++ b/daemon/core/gui/dialogs/wlanconfig.py @@ -4,10 +4,10 @@ import grpc +from core.api.grpc.wrappers import ConfigOption, Node from core.gui.dialogs.dialog import Dialog from core.gui.themes import PADX, PADY from core.gui.widgets import ConfigFrame -from core.gui.wrappers import ConfigOption, Node if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/frames/link.py b/daemon/core/gui/frames/link.py index 339c39f0c..086f7ca87 100644 --- a/daemon/core/gui/frames/link.py +++ b/daemon/core/gui/frames/link.py @@ -1,9 +1,9 @@ import tkinter as tk from typing import TYPE_CHECKING, Optional +from core.api.grpc.wrappers import Interface from core.gui.frames.base import DetailsFrame, InfoFrameBase from core.gui.utils import bandwidth_text -from core.gui.wrappers import Interface if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/frames/node.py b/daemon/core/gui/frames/node.py index 394ecd851..5508784de 100644 --- a/daemon/core/gui/frames/node.py +++ b/daemon/core/gui/frames/node.py @@ -1,9 +1,9 @@ import tkinter as tk from typing import TYPE_CHECKING +from core.api.grpc.wrappers import NodeType from core.gui.frames.base import DetailsFrame, InfoFrameBase from core.gui.nodeutils import NodeUtils -from core.gui.wrappers import NodeType if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/graph/edges.py b/daemon/core/gui/graph/edges.py index d94d47d9d..f9d018244 100644 --- a/daemon/core/gui/graph/edges.py +++ b/daemon/core/gui/graph/edges.py @@ -3,13 +3,13 @@ import tkinter as tk from typing import TYPE_CHECKING, Optional, Tuple +from core.api.grpc.wrappers import Interface, Link from core.gui import themes from core.gui.dialogs.linkconfig import LinkConfigurationDialog from core.gui.frames.link import EdgeInfoFrame, WirelessEdgeInfoFrame from core.gui.graph import tags from core.gui.nodeutils import NodeUtils from core.gui.utils import bandwidth_text, delay_jitter_text -from core.gui.wrappers import Interface, Link if TYPE_CHECKING: from core.gui.graph.graph import CanvasGraph diff --git a/daemon/core/gui/graph/graph.py b/daemon/core/gui/graph/graph.py index cbf3fbb2a..1021e305d 100644 --- a/daemon/core/gui/graph/graph.py +++ b/daemon/core/gui/graph/graph.py @@ -7,6 +7,14 @@ from PIL import Image from PIL.ImageTk import PhotoImage +from core.api.grpc.wrappers import ( + Interface, + Link, + LinkType, + Node, + Session, + ThroughputsEvent, +) from core.gui.dialogs.shapemod import ShapeDialog from core.gui.graph import tags from core.gui.graph.edges import ( @@ -22,7 +30,6 @@ from core.gui.graph.shapeutils import ShapeType, is_draw_shape, is_marker from core.gui.images import ImageEnum, TypeToImage from core.gui.nodeutils import NodeDraw, NodeUtils -from core.gui.wrappers import Interface, Link, LinkType, Node, Session, ThroughputsEvent if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/graph/node.py b/daemon/core/gui/graph/node.py index e63a8b80c..b8172e1dd 100644 --- a/daemon/core/gui/graph/node.py +++ b/daemon/core/gui/graph/node.py @@ -7,6 +7,7 @@ import grpc from PIL.ImageTk import PhotoImage +from core.api.grpc.wrappers import Interface, Node, NodeType from core.gui import nodeutils, themes from core.gui.dialogs.emaneconfig import EmaneConfigDialog from core.gui.dialogs.mobilityconfig import MobilityConfigDialog @@ -20,7 +21,6 @@ from core.gui.graph.tooltip import CanvasTooltip from core.gui.images import ImageEnum, Images from core.gui.nodeutils import ANTENNA_SIZE, NodeUtils -from core.gui.wrappers import Interface, Node, NodeType if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/images.py b/daemon/core/gui/images.py index 0a2f4d5d5..66d92d30c 100644 --- a/daemon/core/gui/images.py +++ b/daemon/core/gui/images.py @@ -5,8 +5,8 @@ from PIL import Image from PIL.ImageTk import PhotoImage +from core.api.grpc.wrappers import NodeType from core.gui.appconfig import LOCAL_ICONS_PATH -from core.gui.wrappers import NodeType class Images: diff --git a/daemon/core/gui/interface.py b/daemon/core/gui/interface.py index f5b1461ed..c075b95ae 100644 --- a/daemon/core/gui/interface.py +++ b/daemon/core/gui/interface.py @@ -4,9 +4,9 @@ import netaddr from netaddr import EUI, IPNetwork +from core.api.grpc.wrappers import Interface, Link, Node from core.gui.graph.node import CanvasNode from core.gui.nodeutils import NodeUtils -from core.gui.wrappers import Interface, Link, Node if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/nodeutils.py b/daemon/core/gui/nodeutils.py index 8cba5bf05..ded1ac89a 100644 --- a/daemon/core/gui/nodeutils.py +++ b/daemon/core/gui/nodeutils.py @@ -3,9 +3,9 @@ from PIL.ImageTk import PhotoImage +from core.api.grpc.wrappers import Node, NodeType from core.gui.appconfig import CustomNode, GuiConfig from core.gui.images import ImageEnum, Images, TypeToImage -from core.gui.wrappers import Node, NodeType ICON_SIZE: int = 48 ANTENNA_SIZE: int = 32 diff --git a/daemon/core/gui/statusbar.py b/daemon/core/gui/statusbar.py index 518a82f94..25f5f9725 100644 --- a/daemon/core/gui/statusbar.py +++ b/daemon/core/gui/statusbar.py @@ -5,9 +5,9 @@ from tkinter import ttk from typing import TYPE_CHECKING, List, Optional +from core.api.grpc.wrappers import ExceptionEvent, ExceptionLevel from core.gui.dialogs.alerts import AlertsDialog from core.gui.themes import Styles -from core.gui.wrappers import ExceptionEvent, ExceptionLevel if TYPE_CHECKING: from core.gui.app import Application diff --git a/daemon/core/gui/widgets.py b/daemon/core/gui/widgets.py index eff1a2a34..004aa7b7e 100644 --- a/daemon/core/gui/widgets.py +++ b/daemon/core/gui/widgets.py @@ -5,10 +5,10 @@ from tkinter import filedialog, font, ttk from typing import TYPE_CHECKING, Any, Callable, Dict, Set, Type +from core.api.grpc.wrappers import ConfigOption, ConfigOptionType from core.gui import appconfig, themes, validation from core.gui.dialogs.dialog import Dialog from core.gui.themes import FRAME_PAD, PADX, PADY -from core.gui.wrappers import ConfigOption, ConfigOptionType if TYPE_CHECKING: from core.gui.app import Application From 570ad9522c0e3b57f9310a7c0f1fc92375950f47 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 1 Sep 2020 16:19:01 -0700 Subject: [PATCH 04/36] initial code for a wrapped grpc client, fix for pygui node emane config, fix for xml reading emane configs specific to nodes/interfaces, fix for adding emane nodes and setting the emane model properly --- daemon/core/api/grpc/clientw.py | 1411 +++++++++++++++++++++++++ daemon/core/api/grpc/server.py | 6 +- daemon/core/api/grpc/wrappers.py | 131 ++- daemon/core/config.py | 2 +- daemon/core/emane/emanemanager.py | 7 +- daemon/core/emulator/session.py | 7 +- daemon/core/gui/coreclient.py | 2 - daemon/core/gui/dialogs/nodeconfig.py | 4 +- daemon/core/xml/corexml.py | 32 +- 9 files changed, 1583 insertions(+), 19 deletions(-) create mode 100644 daemon/core/api/grpc/clientw.py diff --git a/daemon/core/api/grpc/clientw.py b/daemon/core/api/grpc/clientw.py new file mode 100644 index 000000000..10de850b3 --- /dev/null +++ b/daemon/core/api/grpc/clientw.py @@ -0,0 +1,1411 @@ +""" +gRpc client for interfacing with CORE. +""" + +import logging +import threading +from contextlib import contextmanager +from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple + +import grpc + +from core.api.grpc import ( + configservices_pb2, + core_pb2, + core_pb2_grpc, + emane_pb2, + mobility_pb2, + services_pb2, + wlan_pb2, + wrappers, +) +from core.api.grpc.configservices_pb2 import ( + GetConfigServiceDefaultsRequest, + GetConfigServicesRequest, + GetNodeConfigServiceConfigsRequest, + GetNodeConfigServiceRequest, + GetNodeConfigServicesRequest, + SetNodeConfigServiceRequest, +) +from core.api.grpc.core_pb2 import ExecuteScriptRequest +from core.api.grpc.emane_pb2 import ( + EmaneLinkRequest, + EmanePathlossesRequest, + GetEmaneConfigRequest, + GetEmaneEventChannelRequest, + GetEmaneModelConfigRequest, + GetEmaneModelConfigsRequest, + GetEmaneModelsRequest, + SetEmaneConfigRequest, + SetEmaneModelConfigRequest, +) +from core.api.grpc.mobility_pb2 import ( + GetMobilityConfigRequest, + GetMobilityConfigsRequest, + MobilityActionRequest, + MobilityConfig, + SetMobilityConfigRequest, +) +from core.api.grpc.services_pb2 import ( + GetNodeServiceConfigsRequest, + GetNodeServiceFileRequest, + GetNodeServiceRequest, + GetServiceDefaultsRequest, + GetServicesRequest, + ServiceActionRequest, + ServiceDefaults, + ServiceFileConfig, + SetNodeServiceFileRequest, + SetNodeServiceRequest, + SetServiceDefaultsRequest, +) +from core.api.grpc.wlan_pb2 import ( + GetWlanConfigRequest, + GetWlanConfigsRequest, + SetWlanConfigRequest, + WlanConfig, + WlanLinkRequest, +) +from core.emulator.data import IpPrefixes + + +class InterfaceHelper: + """ + Convenience class to help generate IP4 and IP6 addresses for gRPC clients. + """ + + def __init__(self, ip4_prefix: str = None, ip6_prefix: str = None) -> None: + """ + Creates an InterfaceHelper object. + + :param ip4_prefix: ip4 prefix to use for generation + :param ip6_prefix: ip6 prefix to use for generation + :raises ValueError: when both ip4 and ip6 prefixes have not been provided + """ + self.prefixes: IpPrefixes = IpPrefixes(ip4_prefix, ip6_prefix) + + def create_iface( + self, node_id: int, iface_id: int, name: str = None, mac: str = None + ) -> wrappers.Interface: + """ + Create an interface protobuf object. + + :param node_id: node id to create interface for + :param iface_id: interface id + :param name: name of interface + :param mac: mac address for interface + :return: interface protobuf + """ + iface_data = self.prefixes.gen_iface(node_id, name, mac) + return wrappers.Interface( + id=iface_id, + name=iface_data.name, + ip4=iface_data.ip4, + ip4_mask=iface_data.ip4_mask, + ip6=iface_data.ip6, + ip6_mask=iface_data.ip6_mask, + mac=iface_data.mac, + ) + + +def stream_listener(stream: Any, handler: Callable[[core_pb2.Event], None]) -> None: + """ + Listen for stream events and provide them to the handler. + + :param stream: grpc stream that will provide events + :param handler: function that handles an event + :return: nothing + """ + try: + for event in stream: + handler(event) + except grpc.RpcError as e: + if e.code() == grpc.StatusCode.CANCELLED: + logging.debug("stream closed") + else: + logging.exception("stream error") + + +def start_streamer(stream: Any, handler: Callable[[core_pb2.Event], None]) -> None: + """ + Convenience method for starting a grpc stream thread for handling streamed events. + + :param stream: grpc stream that will provide events + :param handler: function that handles an event + :return: nothing + """ + thread = threading.Thread( + target=stream_listener, args=(stream, handler), daemon=True + ) + thread.start() + + +class CoreGrpcClient: + """ + Provides convenience methods for interfacing with the CORE grpc server. + """ + + def __init__(self, address: str = "localhost:50051", proxy: bool = False) -> None: + """ + Creates a CoreGrpcClient instance. + + :param address: grpc server address to connect to + """ + self.address: str = address + self.stub: Optional[core_pb2_grpc.CoreApiStub] = None + self.channel: Optional[grpc.Channel] = None + self.proxy: bool = proxy + + def start_session( + self, session: wrappers.Session, asymmetric_links: List[wrappers.Link] = None + ) -> Tuple[bool, List[str]]: + """ + Start a session. + + :param session: session to start + :param asymmetric_links: link configuration for asymmetric links + :return: tuple of result and exception strings + """ + nodes = [x.to_proto() for x in session.nodes.values()] + links = [x.to_proto() for x in session.links] + if asymmetric_links: + asymmetric_links = [x.to_proto() for x in asymmetric_links] + hooks = [x.to_proto() for x in session.hooks.values()] + emane_config = {k: v.value for k, v in session.emane_config.items()} + emane_model_configs = [] + mobility_configs = [] + wlan_configs = [] + service_configs = [] + service_file_configs = [] + config_service_configs = [] + for node in session.nodes.values(): + for key, config in node.emane_model_configs.items(): + model, iface_id = key + config = wrappers.ConfigOption.to_dict(config) + if iface_id is None: + iface_id = -1 + emane_model_config = emane_pb2.EmaneModelConfig( + node_id=node.id, iface_id=iface_id, model=model, config=config + ) + emane_model_configs.append(emane_model_config) + if node.wlan_config: + config = wrappers.ConfigOption.to_dict(node.wlan_config) + wlan_config = wlan_pb2.WlanConfig(node_id=node.id, config=config) + wlan_configs.append(wlan_config) + if node.mobility_config: + config = wrappers.ConfigOption.to_dict(node.mobility_config) + mobility_config = mobility_pb2.MobilityConfig( + node_id=node.id, config=config + ) + mobility_configs.append(mobility_config) + for name, config in node.service_configs.items(): + service_config = services_pb2.ServiceConfig( + node_id=node.id, + service=name, + directories=config.dirs, + files=config.configs, + startup=config.startup, + validate=config.validate, + shutdown=config.shutdown, + ) + service_configs.append(service_config) + for service, file_configs in node.service_file_configs.items(): + for file, data in file_configs.items(): + service_file_config = services_pb2.ServiceFileConfig( + node_id=node.id, service=service, file=file, data=data + ) + service_file_configs.append(service_file_config) + for name, service_config in node.config_service_configs.items(): + config_service_config = configservices_pb2.ConfigServiceConfig( + node_id=node.id, + name=name, + templates=service_config.templates, + config=service_config.config, + ) + config_service_configs.append(config_service_config) + request = core_pb2.StartSessionRequest( + session_id=session.id, + nodes=nodes, + links=links, + location=session.location.to_proto(), + hooks=hooks, + emane_config=emane_config, + emane_model_configs=emane_model_configs, + wlan_configs=wlan_configs, + mobility_configs=mobility_configs, + service_configs=service_configs, + service_file_configs=service_file_configs, + asymmetric_links=asymmetric_links, + config_service_configs=config_service_configs, + ) + response = self.stub.StartSession(request) + return response.result, list(response.exceptions) + + def stop_session(self, session_id: int) -> bool: + """ + Stop a running session. + + :param session_id: id of session + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.StopSessionRequest(session_id=session_id) + response = self.stub.StopSession(request) + return response.result + + def create_session(self, session_id: int = None) -> int: + """ + Create a session. + + :param session_id: id for session, default is None and one will be created + for you + :return: session id + """ + request = core_pb2.CreateSessionRequest(session_id=session_id) + response = self.stub.CreateSession(request) + return response.session_id + + def delete_session(self, session_id: int) -> bool: + """ + Delete a session. + + :param session_id: id of session + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.DeleteSessionRequest(session_id=session_id) + response = self.stub.DeleteSession(request) + return response.result + + def get_sessions(self) -> List[wrappers.SessionSummary]: + """ + Retrieves all currently known sessions. + + :return: response with a list of currently known session, their state and + number of nodes + """ + response = self.stub.GetSessions(core_pb2.GetSessionsRequest()) + sessions = [] + for session_proto in response.sessions: + session = wrappers.SessionSummary.from_proto(session_proto) + sessions.append(session) + return sessions + + def check_session(self, session_id: int) -> bool: + """ + Check if a session exists. + + :param session_id: id of session to check for + :return: True if exists, False otherwise + """ + request = core_pb2.CheckSessionRequest(session_id=session_id) + response = self.stub.CheckSession(request) + return response.result + + def get_session(self, session_id: int) -> wrappers.Session: + """ + Retrieve a session. + + :param session_id: id of session + :return: session + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.GetSessionRequest(session_id=session_id) + response = self.stub.GetSession(request) + return wrappers.Session.from_proto(response.session) + + def get_session_options(self, session_id: int) -> Dict[str, wrappers.ConfigOption]: + """ + Retrieve session options as a dict with id mapping. + + :param session_id: id of session + :return: session configuration options + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.GetSessionOptionsRequest(session_id=session_id) + response = self.stub.GetSessionOptions(request) + return wrappers.ConfigOption.from_dict(response.config) + + def set_session_options(self, session_id: int, config: Dict[str, str]) -> bool: + """ + Set options for a session. + + :param session_id: id of session + :param config: configuration values to set + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.SetSessionOptionsRequest( + session_id=session_id, config=config + ) + response = self.stub.SetSessionOptions(request) + return response.result + + def get_session_metadata(self, session_id: int) -> Dict[str, str]: + """ + Retrieve session metadata as a dict with id mapping. + + :param session_id: id of session + :return: response with metadata dict + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.GetSessionMetadataRequest(session_id=session_id) + response = self.stub.GetSessionMetadata(request) + return dict(response.config) + + def set_session_metadata(self, session_id: int, config: Dict[str, str]) -> bool: + """ + Set metadata for a session. + + :param session_id: id of session + :param config: configuration values to set + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.SetSessionMetadataRequest( + session_id=session_id, config=config + ) + response = self.stub.SetSessionMetadata(request) + return response.result + + def get_session_location(self, session_id: int) -> wrappers.SessionLocation: + """ + Get session location. + + :param session_id: id of session + :return: response with session position reference and scale + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.GetSessionLocationRequest(session_id=session_id) + response = self.stub.GetSessionLocation(request) + return wrappers.SessionLocation.from_proto(response.location) + + def set_session_location( + self, session_id: int, location: wrappers.SessionLocation + ) -> bool: + """ + Set session location. + + :param session_id: id of session + :param location: session location + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.SetSessionLocationRequest( + session_id=session_id, location=location.to_proto() + ) + response = self.stub.SetSessionLocation(request) + return response.result + + def set_session_state(self, session_id: int, state: wrappers.SessionState) -> bool: + """ + Set session state. + + :param session_id: id of session + :param state: session state to transition to + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.SetSessionStateRequest( + session_id=session_id, state=state.value + ) + response = self.stub.SetSessionState(request) + return response.result + + def set_session_user(self, session_id: int, user: str) -> bool: + """ + Set session user, used for helping to find files without full paths. + + :param session_id: id of session + :param user: user to set for session + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.SetSessionUserRequest(session_id=session_id, user=user) + response = self.stub.SetSessionUser(request) + return response.result + + def add_session_server(self, session_id: int, name: str, host: str) -> bool: + """ + Add distributed session server. + + :param session_id: id of session + :param name: name of server to add + :param host: host address to connect to + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.AddSessionServerRequest( + session_id=session_id, name=name, host=host + ) + response = self.stub.AddSessionServer(request) + return response.result + + def alert( + self, + session_id: int, + level: wrappers.ExceptionLevel, + source: str, + text: str, + node_id: int = None, + ) -> bool: + """ + Initiate an alert to be broadcast out to all listeners. + + :param session_id: id of session + :param level: alert level + :param source: source of alert + :param text: alert text + :param node_id: node associated with alert + :return: True for success, False otherwise + """ + request = core_pb2.SessionAlertRequest( + session_id=session_id, + level=level.value, + source=source, + text=text, + node_id=node_id, + ) + response = self.stub.SessionAlert(request) + return response.result + + # TODO: determine best path for handling non proto events + def events( + self, + session_id: int, + handler: Callable[[core_pb2.Event], None], + events: List[core_pb2.Event] = None, + ) -> grpc.Future: + """ + Listen for session events. + + :param session_id: id of session + :param handler: handler for received events + :param events: events to listen to, defaults to all + :return: stream processing events, can be used to cancel stream + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.EventsRequest(session_id=session_id, events=events) + stream = self.stub.Events(request) + start_streamer(stream, handler) + return stream + + # TODO: determine best path for handling non proto events + def throughputs( + self, session_id: int, handler: Callable[[core_pb2.ThroughputsEvent], None] + ) -> grpc.Future: + """ + Listen for throughput events with information for interfaces and bridges. + + :param session_id: session id + :param handler: handler for every event + :return: stream processing events, can be used to cancel stream + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.ThroughputsRequest(session_id=session_id) + stream = self.stub.Throughputs(request) + start_streamer(stream, handler) + return stream + + # TODO: determine best path for handling non proto events + def cpu_usage( + self, delay: int, handler: Callable[[core_pb2.CpuUsageEvent], None] + ) -> grpc.Future: + """ + Listen for cpu usage events with the given repeat delay. + + :param delay: delay between receiving events + :param handler: handler for every event + :return: stream processing events, can be used to cancel stream + """ + request = core_pb2.CpuUsageRequest(delay=delay) + stream = self.stub.CpuUsage(request) + start_streamer(stream, handler) + return stream + + def add_node(self, session_id: int, node: wrappers.Node, source: str = None) -> int: + """ + Add node to session. + + :param session_id: session id + :param node: node to add + :param source: source application + :return: id of added node + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.AddNodeRequest( + session_id=session_id, node=node.to_proto(), source=source + ) + response = self.stub.AddNode(request) + return response.node_id + + def get_node( + self, session_id: int, node_id: int + ) -> Tuple[wrappers.Node, List[wrappers.Interface]]: + """ + Get node details. + + :param session_id: session id + :param node_id: node id + :return: tuple of node and its interfaces + :raises grpc.RpcError: when session or node doesn't exist + """ + request = core_pb2.GetNodeRequest(session_id=session_id, node_id=node_id) + response = self.stub.GetNode(request) + node = wrappers.Node.from_proto(response.node) + ifaces = [] + for iface_proto in response.ifaces: + iface = wrappers.Interface.from_proto(iface_proto) + ifaces.append(iface) + return node, ifaces + + def edit_node( + self, + session_id: int, + node_id: int, + position: wrappers.Position = None, + icon: str = None, + geo: wrappers.Geo = None, + source: str = None, + ) -> bool: + """ + Edit a node's icon and/or location, can only use position(x,y) or + geo(lon, lat, alt), not both. + + :param session_id: session id + :param node_id: node id + :param position: x,y location for node + :param icon: path to icon for gui to use for node + :param geo: lon,lat,alt location for node + :param source: application source + :return: True for success, False otherwise + :raises grpc.RpcError: when session or node doesn't exist + """ + request = core_pb2.EditNodeRequest( + session_id=session_id, + node_id=node_id, + position=position.to_proto(), + icon=icon, + source=source, + geo=geo.to_proto(), + ) + response = self.stub.EditNode(request) + return response.result + + # TODO: determine path to stream non proto requests + def move_nodes(self, move_iterator: Iterable[core_pb2.MoveNodesRequest]) -> None: + """ + Stream node movements using the provided iterator. + + :param move_iterator: iterator for generating node movements + :return: nothing + :raises grpc.RpcError: when session or nodes do not exist + """ + self.stub.MoveNodes(move_iterator) + + def delete_node(self, session_id: int, node_id: int, source: str = None) -> bool: + """ + Delete node from session. + + :param session_id: session id + :param node_id: node id + :param source: application source + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.DeleteNodeRequest( + session_id=session_id, node_id=node_id, source=source + ) + response = self.stub.DeleteNode(request) + return response.result + + def node_command( + self, + session_id: int, + node_id: int, + command: str, + wait: bool = True, + shell: bool = False, + ) -> Tuple[int, str]: + """ + Send command to a node and get the output. + + :param session_id: session id + :param node_id: node id + :param command: command to run on node + :param wait: wait for command to complete + :param shell: send shell command + :return: returns tuple of return code and output + :raises grpc.RpcError: when session or node doesn't exist + """ + request = core_pb2.NodeCommandRequest( + session_id=session_id, + node_id=node_id, + command=command, + wait=wait, + shell=shell, + ) + response = self.stub.NodeCommand(request) + return response.return_code, response.output + + def get_node_terminal(self, session_id: int, node_id: int) -> str: + """ + Retrieve terminal command string for launching a local terminal. + + :param session_id: session id + :param node_id: node id + :return: node terminal + :raises grpc.RpcError: when session or node doesn't exist + """ + request = core_pb2.GetNodeTerminalRequest( + session_id=session_id, node_id=node_id + ) + response = self.stub.GetNodeTerminal(request) + return response.terminal + + def get_node_links(self, session_id: int, node_id: int) -> List[wrappers.Link]: + """ + Get current links for a node. + + :param session_id: session id + :param node_id: node id + :return: list of links + :raises grpc.RpcError: when session or node doesn't exist + """ + request = core_pb2.GetNodeLinksRequest(session_id=session_id, node_id=node_id) + response = self.stub.GetNodeLinks(request) + links = [] + for link_proto in response.links: + link = wrappers.Link.from_proto(link_proto) + links.append(link) + return links + + def add_link( + self, session_id: int, link: wrappers.Link, source: str = None + ) -> Tuple[bool, wrappers.Interface, wrappers.Interface]: + """ + Add a link between nodes. + + :param session_id: session id + :param link: link to add + :param source: application source + :return: tuple of result and finalized interface values + :raises grpc.RpcError: when session or one of the nodes don't exist + """ + request = core_pb2.AddLinkRequest( + session_id=session_id, link=link.to_proto(), source=source + ) + response = self.stub.AddLink(request) + iface1 = wrappers.Interface.from_proto(response.iface1) + iface2 = wrappers.Interface.from_proto(response.iface2) + return response.result, iface1, iface2 + + def edit_link( + self, session_id: int, link: wrappers.Link, source: str = None + ) -> bool: + """ + Edit a link between nodes. + + :param session_id: session id + :param link: link to edit + :param source: application source + :return: response with result of success or failure + :raises grpc.RpcError: when session or one of the nodes don't exist + """ + iface1_id = link.iface1.id if link.iface1 else None + iface2_id = link.iface2.id if link.iface2 else None + request = core_pb2.EditLinkRequest( + session_id=session_id, + node1_id=link.node1_id, + node2_id=link.node2_id, + options=link.options.to_proto(), + iface1_id=iface1_id, + iface2_id=iface2_id, + source=source, + ) + response = self.stub.EditLink(request) + return response.result + + def delete_link( + self, session_id: int, link: wrappers.Link, source: str = None + ) -> bool: + """ + Delete a link between nodes. + + :param session_id: session id + :param link: link to delete + :param source: application source + :return: response with result of success or failure + :raises grpc.RpcError: when session doesn't exist + """ + iface1_id = link.iface1.id if link.iface1 else None + iface2_id = link.iface2.id if link.iface2 else None + request = core_pb2.DeleteLinkRequest( + session_id=session_id, + node1_id=link.node1_id, + node2_id=link.node2_id, + iface1_id=iface1_id, + iface2_id=iface2_id, + source=source, + ) + response = self.stub.DeleteLink(request) + return response.result + + def get_hooks(self, session_id: int) -> List[wrappers.Hook]: + """ + Get all hook scripts. + + :param session_id: session id + :return: list of hooks + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.GetHooksRequest(session_id=session_id) + response = self.stub.GetHooks(request) + hooks = [] + for hook_proto in response.hooks: + hook = wrappers.Hook.from_proto(hook_proto) + hooks.append(hook) + return hooks + + def add_hook( + self, + session_id: int, + state: wrappers.SessionState, + file_name: str, + file_data: str, + ) -> bool: + """ + Add hook scripts. + + :param session_id: session id + :param state: state to trigger hook + :param file_name: name of file for hook script + :param file_data: hook script contents + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + hook = core_pb2.Hook(state=state.value, file=file_name, data=file_data) + request = core_pb2.AddHookRequest(session_id=session_id, hook=hook) + response = self.stub.AddHook(request) + return response.result + + def get_mobility_configs( + self, session_id: int + ) -> Dict[int, Dict[str, wrappers.ConfigOption]]: + """ + Get all mobility configurations. + + :param session_id: session id + :return: dict of node id to mobility configuration dict + :raises grpc.RpcError: when session doesn't exist + """ + request = GetMobilityConfigsRequest(session_id=session_id) + response = self.stub.GetMobilityConfigs(request) + configs = {} + for node_id, mapped_config in response.configs.items(): + configs[node_id] = wrappers.ConfigOption.from_dict(mapped_config.config) + return configs + + def get_mobility_config( + self, session_id: int, node_id: int + ) -> Dict[str, wrappers.ConfigOption]: + """ + Get mobility configuration for a node. + + :param session_id: session id + :param node_id: node id + :return: dict of config name to options + :raises grpc.RpcError: when session or node doesn't exist + """ + request = GetMobilityConfigRequest(session_id=session_id, node_id=node_id) + response = self.stub.GetMobilityConfig(request) + return wrappers.ConfigOption.from_dict(response.config) + + def set_mobility_config( + self, session_id: int, node_id: int, config: Dict[str, str] + ) -> bool: + """ + Set mobility configuration for a node. + + :param session_id: session id + :param node_id: node id + :param config: mobility configuration + :return: True for success, False otherwise + :raises grpc.RpcError: when session or node doesn't exist + """ + mobility_config = MobilityConfig(node_id=node_id, config=config) + request = SetMobilityConfigRequest( + session_id=session_id, mobility_config=mobility_config + ) + response = self.stub.SetMobilityConfig(request) + return response.result + + def mobility_action( + self, session_id: int, node_id: int, action: wrappers.MobilityAction + ) -> bool: + """ + Send a mobility action for a node. + + :param session_id: session id + :param node_id: node id + :param action: action to take + :return: True for success, False otherwise + :raises grpc.RpcError: when session or node doesn't exist + """ + request = MobilityActionRequest( + session_id=session_id, node_id=node_id, action=action.value + ) + response = self.stub.MobilityAction(request) + return response.result + + def get_services(self) -> List[wrappers.Service]: + """ + Get all currently loaded services. + + :return: list of services, name and groups only + """ + request = GetServicesRequest() + response = self.stub.GetServices(request) + services = [] + for service_proto in response.services: + service = wrappers.Service.from_proto(service_proto) + services.append(service) + return services + + def get_service_defaults(self, session_id: int) -> List[wrappers.ServiceDefault]: + """ + Get default services for different default node models. + + :param session_id: session id + :return: list of service defaults + :raises grpc.RpcError: when session doesn't exist + """ + request = GetServiceDefaultsRequest(session_id=session_id) + response = self.stub.GetServiceDefaults(request) + defaults = [] + for default_proto in response.defaults: + default = wrappers.ServiceDefault.from_proto(default_proto) + defaults.append(default) + return defaults + + def set_service_defaults( + self, session_id: int, service_defaults: Dict[str, List[str]] + ) -> bool: + """ + Set default services for node models. + + :param session_id: session id + :param service_defaults: node models to lists of services + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + defaults = [] + for node_type in service_defaults: + services = service_defaults[node_type] + default = ServiceDefaults(node_type=node_type, services=services) + defaults.append(default) + request = SetServiceDefaultsRequest(session_id=session_id, defaults=defaults) + response = self.stub.SetServiceDefaults(request) + return response.result + + def get_node_service_configs( + self, session_id: int + ) -> List[wrappers.NodeServiceData]: + """ + Get service data for a node. + + :param session_id: session id + :return: list of node service data + :raises grpc.RpcError: when session doesn't exist + """ + request = GetNodeServiceConfigsRequest(session_id=session_id) + response = self.stub.GetNodeServiceConfigs(request) + node_services = [] + for service_proto in response.configs: + node_service = wrappers.NodeServiceData.from_proto(service_proto) + node_services.append(node_service) + return node_services + + def get_node_service( + self, session_id: int, node_id: int, service: str + ) -> wrappers.NodeServiceData: + """ + Get service data for a node. + + :param session_id: session id + :param node_id: node id + :param service: service name + :return: node service data + :raises grpc.RpcError: when session or node doesn't exist + """ + request = GetNodeServiceRequest( + session_id=session_id, node_id=node_id, service=service + ) + response = self.stub.GetNodeService(request) + return wrappers.NodeServiceData.from_proto(response.service) + + def get_node_service_file( + self, session_id: int, node_id: int, service: str, file_name: str + ) -> str: + """ + Get a service file for a node. + + :param session_id: session id + :param node_id: node id + :param service: service name + :param file_name: file name to get data for + :return: file data + :raises grpc.RpcError: when session or node doesn't exist + """ + request = GetNodeServiceFileRequest( + session_id=session_id, node_id=node_id, service=service, file=file_name + ) + response = self.stub.GetNodeServiceFile(request) + return response.data + + def set_node_service( + self, session_id: int, service_config: wrappers.ServiceConfig + ) -> bool: + """ + Set service data for a node. + + :param session_id: session id + :param service_config: service configuration for a node + :return: True for success, False otherwise + :raises grpc.RpcError: when session or node doesn't exist + """ + request = SetNodeServiceRequest( + session_id=session_id, config=service_config.to_proto() + ) + response = self.stub.SetNodeService(request) + return response.result + + def set_node_service_file( + self, session_id: int, node_id: int, service: str, file_name: str, data: str + ) -> bool: + """ + Set a service file for a node. + + :param session_id: session id + :param node_id: node id + :param service: service name + :param file_name: file name to save + :param data: data to save for file + :return: True for success, False otherwise + :raises grpc.RpcError: when session or node doesn't exist + """ + config = ServiceFileConfig( + node_id=node_id, service=service, file=file_name, data=data + ) + request = SetNodeServiceFileRequest(session_id=session_id, config=config) + response = self.stub.SetNodeServiceFile(request) + return response.result + + def service_action( + self, + session_id: int, + node_id: int, + service: str, + action: wrappers.ServiceAction, + ) -> bool: + """ + Send an action to a service for a node. + + :param session_id: session id + :param node_id: node id + :param service: service name + :param action: action for service (start, stop, restart, + validate) + :return: True for success, False otherwise + :raises grpc.RpcError: when session or node doesn't exist + """ + request = ServiceActionRequest( + session_id=session_id, node_id=node_id, service=service, action=action.value + ) + response = self.stub.ServiceAction(request) + return response.result + + def get_wlan_configs( + self, session_id: int + ) -> Dict[int, Dict[str, wrappers.ConfigOption]]: + """ + Get all wlan configurations. + + :param session_id: session id + :return: dict of node ids to dict of names to options + :raises grpc.RpcError: when session doesn't exist + """ + request = GetWlanConfigsRequest(session_id=session_id) + response = self.stub.GetWlanConfigs(request) + configs = {} + for node_id, mapped_config in response.configs.items(): + configs[node_id] = wrappers.ConfigOption.from_dict(mapped_config.config) + return configs + + def get_wlan_config( + self, session_id: int, node_id: int + ) -> Dict[str, wrappers.ConfigOption]: + """ + Get wlan configuration for a node. + + :param session_id: session id + :param node_id: node id + :return: dict of names to options + :raises grpc.RpcError: when session doesn't exist + """ + request = GetWlanConfigRequest(session_id=session_id, node_id=node_id) + response = self.stub.GetWlanConfig(request) + return wrappers.ConfigOption.from_dict(response.config) + + def set_wlan_config( + self, session_id: int, node_id: int, config: Dict[str, str] + ) -> bool: + """ + Set wlan configuration for a node. + + :param session_id: session id + :param node_id: node id + :param config: wlan configuration + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + wlan_config = WlanConfig(node_id=node_id, config=config) + request = SetWlanConfigRequest(session_id=session_id, wlan_config=wlan_config) + response = self.stub.SetWlanConfig(request) + return response.result + + def get_emane_config(self, session_id: int) -> Dict[str, wrappers.ConfigOption]: + """ + Get session emane configuration. + + :param session_id: session id + :return: response with a list of configuration groups + :raises grpc.RpcError: when session doesn't exist + """ + request = GetEmaneConfigRequest(session_id=session_id) + response = self.stub.GetEmaneConfig(request) + return wrappers.ConfigOption.from_dict(response.config) + + def set_emane_config(self, session_id: int, config: Dict[str, str]) -> bool: + """ + Set session emane configuration. + + :param session_id: session id + :param config: emane configuration + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = SetEmaneConfigRequest(session_id=session_id, config=config) + response = self.stub.SetEmaneConfig(request) + return response.result + + def get_emane_models(self, session_id: int) -> List[str]: + """ + Get session emane models. + + :param session_id: session id + :return: list of emane models + :raises grpc.RpcError: when session doesn't exist + """ + request = GetEmaneModelsRequest(session_id=session_id) + response = self.stub.GetEmaneModels(request) + return list(response.models) + + def get_emane_model_config( + self, session_id: int, node_id: int, model: str, iface_id: int = -1 + ) -> Dict[str, wrappers.ConfigOption]: + """ + Get emane model configuration for a node or a node's interface. + + :param session_id: session id + :param node_id: node id + :param model: emane model name + :param iface_id: node interface id + :return: dict of names to options + :raises grpc.RpcError: when session doesn't exist + """ + request = GetEmaneModelConfigRequest( + session_id=session_id, node_id=node_id, model=model, iface_id=iface_id + ) + response = self.stub.GetEmaneModelConfig(request) + return wrappers.ConfigOption.from_dict(response.config) + + def set_emane_model_config( + self, session_id: int, emane_model_config: wrappers.EmaneModelConfig + ) -> bool: + """ + Set emane model configuration for a node or a node's interface. + + :param session_id: session id + :param emane_model_config: emane model config to set + :return: True for success, False otherwise + :raises grpc.RpcError: when session doesn't exist + """ + request = SetEmaneModelConfigRequest( + session_id=session_id, emane_model_config=emane_model_config.to_proto() + ) + response = self.stub.SetEmaneModelConfig(request) + return response.result + + def get_emane_model_configs( + self, session_id: int + ) -> List[wrappers.EmaneModelConfig]: + """ + Get all EMANE model configurations for a session. + + :param session_id: session to get emane model configs + :return: list of emane model configs + :raises grpc.RpcError: when session doesn't exist + """ + request = GetEmaneModelConfigsRequest(session_id=session_id) + response = self.stub.GetEmaneModelConfigs(request) + configs = [] + for config_proto in response.configs: + config = wrappers.EmaneModelConfig.from_proto(config_proto) + configs.append(config) + return configs + + def save_xml(self, session_id: int, file_path: str) -> None: + """ + Save the current scenario to an XML file. + + :param session_id: session to save xml file for + :param file_path: local path to save scenario XML file to + :return: nothing + :raises grpc.RpcError: when session doesn't exist + """ + request = core_pb2.SaveXmlRequest(session_id=session_id) + response = self.stub.SaveXml(request) + with open(file_path, "w") as xml_file: + xml_file.write(response.data) + + def open_xml(self, file_path: str, start: bool = False) -> Tuple[bool, int]: + """ + Load a local scenario XML file to open as a new session. + + :param file_path: path of scenario XML file + :param start: tuple of result and session id when successful + :return: response with opened session id + """ + with open(file_path, "r") as xml_file: + data = xml_file.read() + request = core_pb2.OpenXmlRequest(data=data, start=start, file=file_path) + response = self.stub.OpenXml(request) + return response.result, response.session_id + + def emane_link(self, session_id: int, nem1: int, nem2: int, linked: bool) -> bool: + """ + Helps broadcast wireless link/unlink between EMANE nodes. + + :param session_id: session to emane link + :param nem1: first nem for emane link + :param nem2: second nem for emane link + :param linked: True to link, False to unlink + :return: True for success, False otherwise + :raises grpc.RpcError: when session or nodes related to nems do not exist + """ + request = EmaneLinkRequest( + session_id=session_id, nem1=nem1, nem2=nem2, linked=linked + ) + response = self.stub.EmaneLink(request) + return response.result + + def get_ifaces(self) -> List[str]: + """ + Retrieves a list of interfaces available on the host machine that are not + a part of a CORE session. + + :return: list of interfaces + """ + request = core_pb2.GetInterfacesRequest() + response = self.stub.GetInterfaces(request) + return list(response.ifaces) + + def get_config_services(self) -> List[wrappers.ConfigService]: + """ + Retrieve all known config services. + + :return: list of config services + """ + request = GetConfigServicesRequest() + response = self.stub.GetConfigServices(request) + services = [] + for service_proto in response.services: + service = wrappers.ConfigService.from_proto(service_proto) + services.append(service) + return services + + def get_config_service_defaults(self, name: str) -> wrappers.ConfigServiceDefaults: + """ + Retrieves config service default values. + + :param name: name of service to get defaults for + :return: config service defaults + """ + request = GetConfigServiceDefaultsRequest(name=name) + response = self.stub.GetConfigServiceDefaults(request) + return wrappers.ConfigServiceDefaults.from_proto(response) + + def get_node_config_service_configs( + self, session_id: int + ) -> List[wrappers.ConfigServiceConfig]: + """ + Retrieves all node config service configurations for a session. + + :param session_id: session to get config service configurations for + :return: list of node config service configs + :raises grpc.RpcError: when session doesn't exist + """ + request = GetNodeConfigServiceConfigsRequest(session_id=session_id) + response = self.stub.GetNodeConfigServiceConfigs(request) + configs = [] + for config_proto in response.configs: + config = wrappers.ConfigServiceConfig.from_proto(config_proto) + configs.append(config) + return configs + + def get_node_config_service( + self, session_id: int, node_id: int, name: str + ) -> Dict[str, str]: + """ + Retrieves information for a specific config service on a node. + + :param session_id: session node belongs to + :param node_id: id of node to get service information from + :param name: name of service + :return: config dict of names to values + :raises grpc.RpcError: when session or node doesn't exist + """ + request = GetNodeConfigServiceRequest( + session_id=session_id, node_id=node_id, name=name + ) + response = self.stub.GetNodeConfigService(request) + return dict(response.config) + + def get_node_config_services(self, session_id: int, node_id: int) -> List[str]: + """ + Retrieves the config services currently assigned to a node. + + :param session_id: session node belongs to + :param node_id: id of node to get config services for + :return: list of config services + :raises grpc.RpcError: when session or node doesn't exist + """ + request = GetNodeConfigServicesRequest(session_id=session_id, node_id=node_id) + response = self.stub.GetNodeConfigServices(request) + return list(response.services) + + def set_node_config_service( + self, session_id: int, node_id: int, name: str, config: Dict[str, str] + ) -> bool: + """ + Assigns a config service to a node with the provided configuration. + + :param session_id: session node belongs to + :param node_id: id of node to assign config service to + :param name: name of service + :param config: service configuration + :return: True for success, False otherwise + :raises grpc.RpcError: when session or node doesn't exist + """ + request = SetNodeConfigServiceRequest( + session_id=session_id, node_id=node_id, name=name, config=config + ) + response = self.stub.SetNodeConfigService(request) + return response.result + + def get_emane_event_channel(self, session_id: int) -> wrappers.EmaneEventChannel: + """ + Retrieves the current emane event channel being used for a session. + + :param session_id: session to get emane event channel for + :return: emane event channel + :raises grpc.RpcError: when session doesn't exist + """ + request = GetEmaneEventChannelRequest(session_id=session_id) + response = self.stub.GetEmaneEventChannel(request) + return wrappers.EmaneEventChannel.from_proto(response) + + def execute_script(self, script: str) -> Optional[int]: + """ + Executes a python script given context of the current CoreEmu object. + + :param script: script to execute + :return: create session id for script executed + """ + request = ExecuteScriptRequest(script=script) + response = self.stub.ExecuteScript(request) + return response.session_id if response.session_id else None + + def wlan_link( + self, session_id: int, wlan_id: int, node1_id: int, node2_id: int, linked: bool + ) -> bool: + """ + Links/unlinks nodes on the same WLAN. + + :param session_id: session id containing wlan and nodes + :param wlan_id: wlan nodes must belong to + :param node1_id: first node of pair to link/unlink + :param node2_id: second node of pair to link/unlin + :param linked: True to link, False to unlink + :return: True for success, False otherwise + :raises grpc.RpcError: when session or one of the nodes do not exist + """ + request = WlanLinkRequest( + session_id=session_id, + wlan=wlan_id, + node1_id=node1_id, + node2_id=node2_id, + linked=linked, + ) + response = self.stub.WlanLink(request) + return response.result + + # TODO: determine path to stream non proto requests + def emane_pathlosses( + self, pathloss_iterator: Iterable[EmanePathlossesRequest] + ) -> None: + """ + Stream EMANE pathloss events. + + :param pathloss_iterator: iterator for sending emane pathloss events + :return: nothing + :raises grpc.RpcError: when a pathloss event session or one of the nodes do not + exist + """ + self.stub.EmanePathlosses(pathloss_iterator) + + def connect(self) -> None: + """ + Open connection to server, must be closed manually. + + :return: nothing + """ + self.channel = grpc.insecure_channel( + self.address, options=[("grpc.enable_http_proxy", self.proxy)] + ) + self.stub = core_pb2_grpc.CoreApiStub(self.channel) + + def close(self) -> None: + """ + Close currently opened server channel connection. + + :return: nothing + """ + if self.channel: + self.channel.close() + self.channel = None + + @contextmanager + def context_connect(self) -> Generator: + """ + Makes a context manager based connection to the server, will close after + context ends. + + :return: nothing + """ + try: + self.connect() + yield + finally: + self.close() diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index aabf2177e..2b0bc2a83 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -1438,7 +1438,9 @@ def GetEmaneModelConfig( """ logging.debug("get emane model config: %s", request) session = self.get_session(request.session_id, context) - model = session.emane.models[request.model] + model = session.emane.models.get(request.model) + if not model: + raise CoreError(f"invalid emane model: {request.model}") _id = get_emane_model_id(request.node_id, request.iface_id) current_config = session.emane.get_model_config(_id, request.model) config = get_config_options(current_config, model) @@ -1483,7 +1485,7 @@ def SaveXml( self, request: core_pb2.SaveXmlRequest, context: ServicerContext ) -> core_pb2.SaveXmlResponse: """ - Export the session nto the EmulationScript XML format + Export the session into the EmulationScript XML format :param request: save xml request :param context: context object diff --git a/daemon/core/api/grpc/wrappers.py b/daemon/core/api/grpc/wrappers.py index 52384fe2a..285bdfce8 100644 --- a/daemon/core/api/grpc/wrappers.py +++ b/daemon/core/api/grpc/wrappers.py @@ -3,7 +3,13 @@ from pathlib import Path from typing import Dict, List, Optional, Set, Tuple -from core.api.grpc import common_pb2, configservices_pb2, core_pb2, services_pb2 +from core.api.grpc import ( + common_pb2, + configservices_pb2, + core_pb2, + emane_pb2, + services_pb2, +) class ConfigServiceValidationMode(Enum): @@ -87,6 +93,13 @@ class MessageType(Enum): TTY = 64 +class ServiceAction(Enum): + START = 0 + STOP = 1 + RESTART = 2 + VALIDATE = 3 + + @dataclass class ConfigService: group: str @@ -120,12 +133,67 @@ def from_proto(cls, proto: configservices_pb2.ConfigService) -> "ConfigService": ) +@dataclass +class ConfigServiceConfig: + node_id: int + name: str + templates: Dict[str, str] + config: Dict[str, str] + + @classmethod + def from_proto( + cls, proto: configservices_pb2.ConfigServiceConfig + ) -> "ConfigServiceConfig": + return ConfigServiceConfig( + node_id=proto.node_id, + name=proto.name, + templates=dict(proto.templates), + config=dict(proto.config), + ) + + @dataclass class ConfigServiceData: templates: Dict[str, str] = field(default_factory=dict) config: Dict[str, str] = field(default_factory=dict) +@dataclass +class ConfigServiceDefaults: + templates: Dict[str, str] + config: Dict[str, "ConfigOption"] + modes: List[str] + + @classmethod + def from_proto( + cls, proto: configservices_pb2.GetConfigServicesResponse + ) -> "ConfigServiceDefaults": + config = ConfigOption.from_dict(proto.config) + return ConfigServiceDefaults( + templates=dict(proto.templates), config=config, modes=list(proto.modes) + ) + + +@dataclass +class Service: + group: str + name: str + + @classmethod + def from_proto(cls, proto: services_pb2.Service) -> "Service": + return Service(group=proto.group, name=proto.name) + + +@dataclass +class ServiceDefault: + node_type: str + services: List[str] + + @classmethod + def from_proto(cls, proto: services_pb2.ServiceDefaults) -> "ServiceDefault": + return ServiceDefault(node_type=proto.node_type, services=list(proto.services)) + + @dataclass class NodeServiceData: executables: List[str] @@ -155,6 +223,28 @@ def from_proto(cls, proto: services_pb2.NodeServiceData) -> "NodeServiceData": ) +@dataclass +class ServiceConfig: + node_id: int + service: str + files: List[str] = None + directories: List[str] = None + startup: List[str] = None + validate: List[str] = None + shutdown: List[str] = None + + def to_proto(self) -> services_pb2.ServiceConfig: + return services_pb2.ServiceConfig( + node_id=self.node_id, + service=self.service, + files=self.files, + directories=self.directories, + startup=self.startup, + validate=self.validate, + shutdown=self.shutdown, + ) + + @dataclass class BridgeThroughput: node_id: int @@ -471,6 +561,30 @@ def to_proto(self) -> core_pb2.Hook: return core_pb2.Hook(state=self.state.value, file=self.file, data=self.data) +@dataclass +class EmaneModelConfig: + node_id: int + model: str + iface_id: int = -1 + config: Dict[str, ConfigOption] = None + + @classmethod + def from_proto(cls, proto: emane_pb2.GetEmaneModelConfig) -> "EmaneModelConfig": + iface_id = proto.iface_id if proto.iface_id != -1 else None + config = ConfigOption.from_dict(proto.config) + return EmaneModelConfig( + node_id=proto.node_id, iface_id=iface_id, model=proto.model, config=config + ) + + def to_proto(self) -> emane_pb2.EmaneModelConfig: + return emane_pb2.EmaneModelConfig( + node_id=self.node_id, + model=self.model, + iface_id=self.iface_id, + config=self.config, + ) + + @dataclass class Position: x: float @@ -660,3 +774,18 @@ def from_proto(cls, proto: core_pb2.NodeEvent) -> "NodeEvent": message_type=MessageType(proto.message_type), node=Node.from_proto(proto.node), ) + + +@dataclass +class EmaneEventChannel: + group: str + port: int + device: str + + @classmethod + def from_proto( + cls, proto: emane_pb2.GetEmaneEventChannelResponse + ) -> "EmaneEventChannel": + return EmaneEventChannel( + group=proto.group, port=proto.port, device=proto.device + ) diff --git a/daemon/core/config.py b/daemon/core/config.py index 618e1273d..222abf010 100644 --- a/daemon/core/config.py +++ b/daemon/core/config.py @@ -212,7 +212,7 @@ def get_config( def get_configs( self, node_id: int = _default_node, config_type: str = _default_type - ) -> Dict[str, str]: + ) -> Optional[Dict[str, str]]: """ Retrieve configurations for a node and configuration type. diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index ec39137d7..7c1e5b832 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -145,14 +145,17 @@ def get_iface_config( key += iface.node_id # try retrieve interface specific configuration, avoid getting defaults config = self.get_configs(node_id=key, config_type=model_name) - # otherwise retrieve the interfaces node configuration, avoid using defaults + # attempt to retrieve node specific conifg, when iface config is not present if not config: config = self.get_configs(node_id=iface.node.id, config_type=model_name) - # get non interface config, when none found + # attempt to get emane net specific config, when node config is not present if not config: # with EMANE 0.9.2+, we need an extra NEM XML from # model.buildnemxmlfiles(), so defaults are returned here config = self.get_configs(node_id=emane_net.id, config_type=model_name) + # return default config values, when a config is not present + if not config: + config = emane_net.model.default_values() return config def config_reset(self, node_id: int = None) -> None: diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 309f0ac6c..7622127a4 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -545,7 +545,12 @@ def add_node( # ensure default emane configuration if isinstance(node, EmaneNet) and options.emane: - self.emane.set_model_config(_id, options.emane) + model = self.emane.models.get(options.emane) + if not model: + raise CoreError( + f"node({node.name}) emane model({options.emane}) does not exist" + ) + node.setmodel(model, {}) if self.state == EventTypes.RUNTIME_STATE: self.emane.add_node(node) # set default wlan config if needed diff --git a/daemon/core/gui/coreclient.py b/daemon/core/gui/coreclient.py index 116b1617b..f4f8b79a3 100644 --- a/daemon/core/gui/coreclient.py +++ b/daemon/core/gui/coreclient.py @@ -937,8 +937,6 @@ def get_mobility_configs_proto(self) -> List[mobility_pb2.MobilityConfig]: def get_emane_model_configs_proto(self) -> List[emane_pb2.EmaneModelConfig]: configs = [] for node in self.session.nodes.values(): - if node.type != NodeType.EMANE: - continue for key, config in node.emane_model_configs.items(): model, iface_id = key config = ConfigOption.to_dict(config) diff --git a/daemon/core/gui/dialogs/nodeconfig.py b/daemon/core/gui/dialogs/nodeconfig.py index 6327206fa..9ceafa351 100644 --- a/daemon/core/gui/dialogs/nodeconfig.py +++ b/daemon/core/gui/dialogs/nodeconfig.py @@ -282,9 +282,7 @@ def draw_buttons(self) -> None: button.grid(row=0, column=1, sticky=tk.EW) def click_emane_config(self, emane_model: str, iface_id: int) -> None: - dialog = EmaneModelDialog( - self, self.app, self.canvas_node, emane_model, iface_id - ) + dialog = EmaneModelDialog(self, self.app, self.node, emane_model, iface_id) dialog.show() def click_icon(self) -> None: diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index 7e3b35a21..20a302969 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -91,10 +91,14 @@ def create_emane_config(session: "Session") -> etree.Element: def create_emane_model_config( - node_id: int, model: "EmaneModelType", config: Dict[str, str] + node_id: int, + model: "EmaneModelType", + config: Dict[str, str], + iface_id: Optional[int], ) -> etree.Element: emane_element = etree.Element("emane_configuration") add_attribute(emane_element, "node", node_id) + add_attribute(emane_element, "iface", iface_id) add_attribute(emane_element, "model", model.name) mac_element = etree.SubElement(emane_element, "mac") @@ -378,13 +382,19 @@ def write_emane_configs(self) -> None: all_configs = self.session.emane.get_all_configs(node_id) if not all_configs: continue + iface_id = None + if node_id >= 1000: + iface_id = node_id % 1000 + node_id = node_id // 1000 for model_name in all_configs: config = all_configs[model_name] logging.debug( "writing emane config node(%s) model(%s)", node_id, model_name ) model = self.session.emane.models[model_name] - emane_configuration = create_emane_model_config(node_id, model, config) + emane_configuration = create_emane_model_config( + node_id, model, config, iface_id + ) emane_configurations.append(emane_configuration) if emane_configurations.getchildren(): self.scenario.append(emane_configurations) @@ -588,9 +598,9 @@ def read(self, file_name: str) -> None: self.read_mobility_configs() self.read_emane_global_config() self.read_nodes() + self.read_links() self.read_emane_configs() self.read_configservice_configs() - self.read_links() def read_default_services(self) -> None: default_services = self.scenario.find("default_services") @@ -748,6 +758,7 @@ def read_emane_configs(self) -> None: for emane_configuration in emane_configurations.iterchildren(): node_id = get_int(emane_configuration, "node") + iface_id = get_int(emane_configuration, "iface") model_name = emane_configuration.get("model") configs = {} @@ -755,12 +766,13 @@ def read_emane_configs(self) -> None: node = self.session.nodes.get(node_id) if not node: raise CoreXmlError(f"node for emane config doesn't exist: {node_id}") - if not isinstance(node, EmaneNet): - raise CoreXmlError(f"invalid node for emane config: {node.name}") model = self.session.emane.models.get(model_name) if not model: raise CoreXmlError(f"invalid emane model: {model_name}") - node.setmodel(model, {}) + if iface_id is not None and iface_id not in node.ifaces: + raise CoreXmlError( + f"invalid interface id({iface_id}) for node({node.name})" + ) # read and set emane model configuration mac_configuration = emane_configuration.find("mac") @@ -784,7 +796,10 @@ def read_emane_configs(self) -> None: logging.info( "reading emane configuration node(%s) model(%s)", node_id, model_name ) - self.session.emane.set_model_config(node_id, model_name, configs) + key = node_id + if iface_id is not None: + key = node_id * 1000 + iface_id + self.session.emane.set_model_config(key, model_name, configs) def read_mobility_configs(self) -> None: mobility_configurations = self.scenario.find("mobility_configurations") @@ -869,6 +884,9 @@ def read_network(self, network_element: etree.Element) -> None: icon = network_element.get("icon") server = network_element.get("server") options = NodeOptions(name=name, icon=icon, server=server) + if node_type == NodeTypes.EMANE: + model = network_element.get("model") + options.emane = model position_element = network_element.find("position") if position_element is not None: From ba028a2b0036c42b90c921a7889eaf9c56d4357b Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 1 Sep 2020 17:16:05 -0700 Subject: [PATCH 05/36] daemon: just assign emane model, instead of triggering position hooks during non-runtime cases --- daemon/core/emulator/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 7622127a4..2db2c1adc 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -550,7 +550,7 @@ def add_node( raise CoreError( f"node({node.name}) emane model({options.emane}) does not exist" ) - node.setmodel(model, {}) + node.model = model(self, node.id) if self.state == EventTypes.RUNTIME_STATE: self.emane.add_node(node) # set default wlan config if needed From e775ad4c5d86c18ff24b0cc3380ca0e5317e4c8d Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Tue, 1 Sep 2020 17:47:24 -0700 Subject: [PATCH 06/36] fixed invoke task to run emane tests, added emane xml tests for node/interface specific configurations --- daemon/tests/emane/test_emane.py | 124 +++++++++++++++++++++++++++++++ tasks.py | 2 +- 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index f51e30b91..e996f308e 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -161,3 +161,127 @@ def test_xml_emane( assert session.get_node(node2_id, CoreNode) assert session.get_node(emane_id, EmaneNet) assert value == config_value + + def test_xml_emane_node_config( + self, session: Session, tmpdir: TemporaryFile, ip_prefixes: IpPrefixes + ): + # create nodes + options = NodeOptions(model="mdr", x=50, y=50) + node1 = session.add_node(CoreNode, options=options) + iface1_data = ip_prefixes.create_iface(node1) + node2 = session.add_node(CoreNode, options=options) + iface2_data = ip_prefixes.create_iface(node2) + + # create emane node + options = NodeOptions(model=None, emane=EmaneRfPipeModel.name) + emane_node = session.add_node(EmaneNet, options=options) + + # create links + session.add_link(node1.id, emane_node.id, iface1_data) + session.add_link(node2.id, emane_node.id, iface2_data) + + # set node specific conifg + datarate = "101" + session.emane.set_model_config( + node1.id, EmaneRfPipeModel.name, {"datarate": datarate} + ) + + # instantiate session + session.instantiate() + + # save xml + xml_file = tmpdir.join("session.xml") + file_path = xml_file.strpath + session.save_xml(file_path) + + # verify xml file was created and can be parsed + assert xml_file.isfile() + assert ElementTree.parse(file_path) + + # stop current session, clearing data + session.shutdown() + + # verify nodes have been removed from session + with pytest.raises(CoreError): + assert not session.get_node(node1.id, CoreNode) + with pytest.raises(CoreError): + assert not session.get_node(node2.id, CoreNode) + with pytest.raises(CoreError): + assert not session.get_node(emane_node.id, EmaneNet) + + # load saved xml + session.open_xml(file_path, start=True) + + # verify nodes have been recreated + assert session.get_node(node1.id, CoreNode) + assert session.get_node(node2.id, CoreNode) + assert session.get_node(emane_node.id, EmaneNet) + links = [] + for node_id in session.nodes: + node = session.nodes[node_id] + links += node.links() + assert len(links) == 2 + config = session.emane.get_model_config(node1.id, EmaneRfPipeModel.name) + assert config["datarate"] == datarate + + def test_xml_emane_interface_config( + self, session: Session, tmpdir: TemporaryFile, ip_prefixes: IpPrefixes + ): + # create nodes + options = NodeOptions(model="mdr", x=50, y=50) + node1 = session.add_node(CoreNode, options=options) + iface1_data = ip_prefixes.create_iface(node1) + node2 = session.add_node(CoreNode, options=options) + iface2_data = ip_prefixes.create_iface(node2) + + # create emane node + options = NodeOptions(model=None, emane=EmaneRfPipeModel.name) + emane_node = session.add_node(EmaneNet, options=options) + + # create links + session.add_link(node1.id, emane_node.id, iface1_data) + session.add_link(node2.id, emane_node.id, iface2_data) + + # set node specific conifg + datarate = "101" + session.emane.set_model_config( + node1.id * 1000, EmaneRfPipeModel.name, {"datarate": datarate} + ) + + # instantiate session + session.instantiate() + + # save xml + xml_file = tmpdir.join("session.xml") + file_path = xml_file.strpath + session.save_xml(file_path) + + # verify xml file was created and can be parsed + assert xml_file.isfile() + assert ElementTree.parse(file_path) + + # stop current session, clearing data + session.shutdown() + + # verify nodes have been removed from session + with pytest.raises(CoreError): + assert not session.get_node(node1.id, CoreNode) + with pytest.raises(CoreError): + assert not session.get_node(node2.id, CoreNode) + with pytest.raises(CoreError): + assert not session.get_node(emane_node.id, EmaneNet) + + # load saved xml + session.open_xml(file_path, start=True) + + # verify nodes have been recreated + assert session.get_node(node1.id, CoreNode) + assert session.get_node(node2.id, CoreNode) + assert session.get_node(emane_node.id, EmaneNet) + links = [] + for node_id in session.nodes: + node = session.nodes[node_id] + links += node.links() + assert len(links) == 2 + config = session.emane.get_model_config(node1.id * 1000, EmaneRfPipeModel.name) + assert config["datarate"] == datarate diff --git a/tasks.py b/tasks.py index fdf147aee..ab73ccc86 100644 --- a/tasks.py +++ b/tasks.py @@ -495,4 +495,4 @@ def test_emane(c): """ pytest = get_pytest(c) with c.cd(DAEMON_DIR): - c.run(f"{pytest} -v --lf -x tests/emane", pty=True) + c.run(f"sudo {pytest} -v --lf -x tests/emane", pty=True) From a80fda11f571348bb2e740f4ad63ab79e8cc32bf Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 2 Sep 2020 09:44:45 -0700 Subject: [PATCH 07/36] daemon: abstracted out iface specific configuration generation and parsing to common utilities, to avoid duplicate logic and potential differences that may arise --- daemon/core/api/grpc/grpcutils.py | 32 ++--------------------------- daemon/core/api/grpc/server.py | 13 ++++-------- daemon/core/api/tlv/corehandlers.py | 12 +++-------- daemon/core/emane/emanemanager.py | 10 ++++----- daemon/core/utils.py | 32 +++++++++++++++++++++++++++++ daemon/core/xml/corexml.py | 12 ++++------- daemon/tests/emane/test_emane.py | 6 ++++-- 7 files changed, 54 insertions(+), 63 deletions(-) diff --git a/daemon/core/api/grpc/grpcutils.py b/daemon/core/api/grpc/grpcutils.py index eaec23595..7f25f1c16 100644 --- a/daemon/core/api/grpc/grpcutils.py +++ b/daemon/core/api/grpc/grpcutils.py @@ -306,35 +306,6 @@ def get_links(node: NodeBase): return links -def get_emane_model_id(node_id: int, iface_id: int) -> int: - """ - Get EMANE model id - - :param node_id: node id - :param iface_id: interface id - :return: EMANE model id - """ - if iface_id >= 0: - return node_id * 1000 + iface_id - else: - return node_id - - -def parse_emane_model_id(_id: int) -> Tuple[int, int]: - """ - Parses EMANE model id to get true node id and interface id. - - :param _id: id to parse - :return: node id and interface id - """ - iface_id = -1 - node_id = _id - if _id >= 1000: - iface_id = _id % 1000 - node_id = int(_id / 1000) - return node_id, iface_id - - def convert_iface(iface_data: InterfaceData) -> core_pb2.Interface: return core_pb2.Interface( id=iface_data.id, @@ -559,7 +530,8 @@ def get_emane_model_configs(session: Session) -> List[GetEmaneModelConfig]: model = session.emane.models[model_name] current_config = session.emane.get_model_config(_id, model_name) config = get_config_options(current_config, model) - node_id, iface_id = parse_emane_model_id(_id) + node_id, iface_id = utils.parse_iface_config_id(_id) + iface_id = iface_id if iface_id is not None else -1 model_config = GetEmaneModelConfig( node_id=node_id, model=model_name, iface_id=iface_id, config=config ) diff --git a/daemon/core/api/grpc/server.py b/daemon/core/api/grpc/server.py index 2b0bc2a83..55264430e 100644 --- a/daemon/core/api/grpc/server.py +++ b/daemon/core/api/grpc/server.py @@ -56,12 +56,7 @@ SetEmaneModelConfigResponse, ) from core.api.grpc.events import EventStreamer -from core.api.grpc.grpcutils import ( - get_config_options, - get_emane_model_id, - get_links, - get_net_stats, -) +from core.api.grpc.grpcutils import get_config_options, get_links, get_net_stats from core.api.grpc.mobility_pb2 import ( GetMobilityConfigRequest, GetMobilityConfigResponse, @@ -249,7 +244,7 @@ def StartSession( config = session.emane.get_configs() config.update(request.emane_config) for config in request.emane_model_configs: - _id = get_emane_model_id(config.node_id, config.iface_id) + _id = utils.iface_config_id(config.node_id, config.iface_id) session.emane.set_model_config(_id, config.model, config.config) # wlan configs @@ -1441,7 +1436,7 @@ def GetEmaneModelConfig( model = session.emane.models.get(request.model) if not model: raise CoreError(f"invalid emane model: {request.model}") - _id = get_emane_model_id(request.node_id, request.iface_id) + _id = utils.iface_config_id(request.node_id, request.iface_id) current_config = session.emane.get_model_config(_id, request.model) config = get_config_options(current_config, model) return GetEmaneModelConfigResponse(config=config) @@ -1460,7 +1455,7 @@ def SetEmaneModelConfig( logging.debug("set emane model config: %s", request) session = self.get_session(request.session_id, context) model_config = request.emane_model_config - _id = get_emane_model_id(model_config.node_id, model_config.iface_id) + _id = utils.iface_config_id(model_config.node_id, model_config.iface_id) session.emane.set_model_config(_id, model_config.model, model_config.config) return SetEmaneModelConfigResponse(result=True) diff --git a/daemon/core/api/tlv/corehandlers.py b/daemon/core/api/tlv/corehandlers.py index 4bd78d83e..1c1c272d0 100644 --- a/daemon/core/api/tlv/corehandlers.py +++ b/daemon/core/api/tlv/corehandlers.py @@ -1331,9 +1331,7 @@ def handle_config_mobility_models(self, message_type, config_data): iface_id = config_data.iface_id values_str = config_data.data_values - if iface_id is not None: - node_id = node_id * 1000 + iface_id - + node_id = utils.iface_config_id(node_id, iface_id) logging.debug( "received configure message for %s nodenum: %s", object_name, node_id ) @@ -1381,9 +1379,7 @@ def handle_config_emane(self, message_type, config_data): iface_id = config_data.iface_id values_str = config_data.data_values - if iface_id is not None: - node_id = node_id * 1000 + iface_id - + node_id = utils.iface_config_id(node_id, iface_id) logging.debug( "received configure message for %s nodenum: %s", object_name, node_id ) @@ -1413,9 +1409,7 @@ def handle_config_emane_models(self, message_type, config_data): iface_id = config_data.iface_id values_str = config_data.data_values - if iface_id is not None: - node_id = node_id * 1000 + iface_id - + node_id = utils.iface_config_id(node_id, iface_id) logging.debug( "received configure message for %s nodenum: %s", object_name, node_id ) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index 7c1e5b832..a6b299279 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -140,12 +140,12 @@ def get_iface_config( # Adamson change: first check for iface config keyed by "node:iface.name" # (so that nodes w/ multiple interfaces of same conftype can have # different configs for each separate interface) - key = 1000 * iface.node.id + config = None + # try to retrieve interface specific configuration if iface.node_id is not None: - key += iface.node_id - # try retrieve interface specific configuration, avoid getting defaults - config = self.get_configs(node_id=key, config_type=model_name) - # attempt to retrieve node specific conifg, when iface config is not present + key = utils.iface_config_id(iface.node.id, iface.node_id) + config = self.get_configs(node_id=key, config_type=model_name) + # attempt to retrieve node specific config, when iface config is not present if not config: config = self.get_configs(node_id=iface.node.id, config_type=model_name) # attempt to get emane net specific config, when node config is not present diff --git a/daemon/core/utils.py b/daemon/core/utils.py index 459b7d568..40001fe1a 100644 --- a/daemon/core/utils.py +++ b/daemon/core/utils.py @@ -41,6 +41,7 @@ T = TypeVar("T") DEVNULL = open(os.devnull, "wb") +IFACE_CONFIG_FACTOR: int = 1000 def execute_file( @@ -430,3 +431,34 @@ def random_mac() -> str: value |= 0x00163E << 24 mac = netaddr.EUI(value, dialect=netaddr.mac_unix_expanded) return str(mac) + + +def iface_config_id(node_id: int, iface_id: int = None) -> int: + """ + Common utility to generate a configuration id, in case an interface is being + targeted. + + :param node_id: node for config id + :param iface_id: interface for config id + :return: generated config id when interface is present, node id otherwise + """ + if iface_id is not None and iface_id >= 0: + return node_id * IFACE_CONFIG_FACTOR + iface_id + else: + return node_id + + +def parse_iface_config_id(config_id: int) -> Tuple[int, Optional[int]]: + """ + Parses configuration id, that may be potentially derived from an interface for a + node. + + :param config_id: configuration id to parse + :return: + """ + iface_id = None + node_id = config_id + if config_id >= IFACE_CONFIG_FACTOR: + iface_id = config_id % IFACE_CONFIG_FACTOR + node_id = config_id // IFACE_CONFIG_FACTOR + return node_id, iface_id diff --git a/daemon/core/xml/corexml.py b/daemon/core/xml/corexml.py index 20a302969..79d3209d6 100644 --- a/daemon/core/xml/corexml.py +++ b/daemon/core/xml/corexml.py @@ -5,6 +5,7 @@ import core.nodes.base import core.nodes.physical +from core import utils from core.emane.nodes import EmaneNet from core.emulator.data import InterfaceData, LinkData, LinkOptions, NodeOptions from core.emulator.enumerations import EventTypes, NodeTypes @@ -382,10 +383,7 @@ def write_emane_configs(self) -> None: all_configs = self.session.emane.get_all_configs(node_id) if not all_configs: continue - iface_id = None - if node_id >= 1000: - iface_id = node_id % 1000 - node_id = node_id // 1000 + node_id, iface_id = utils.parse_iface_config_id(node_id) for model_name in all_configs: config = all_configs[model_name] logging.debug( @@ -796,10 +794,8 @@ def read_emane_configs(self) -> None: logging.info( "reading emane configuration node(%s) model(%s)", node_id, model_name ) - key = node_id - if iface_id is not None: - key = node_id * 1000 + iface_id - self.session.emane.set_model_config(key, model_name, configs) + node_id = utils.iface_config_id(node_id, iface_id) + self.session.emane.set_model_config(node_id, model_name, configs) def read_mobility_configs(self) -> None: mobility_configurations = self.scenario.find("mobility_configurations") diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index e996f308e..6c21e4d7b 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -8,6 +8,7 @@ import pytest +from core import utils from core.emane.bypass import EmaneBypassModel from core.emane.commeffect import EmaneCommEffectModel from core.emane.emanemodel import EmaneModel @@ -244,8 +245,9 @@ def test_xml_emane_interface_config( # set node specific conifg datarate = "101" + config_id = utils.iface_config_id(node1.id, iface1_data.id) session.emane.set_model_config( - node1.id * 1000, EmaneRfPipeModel.name, {"datarate": datarate} + config_id, EmaneRfPipeModel.name, {"datarate": datarate} ) # instantiate session @@ -283,5 +285,5 @@ def test_xml_emane_interface_config( node = session.nodes[node_id] links += node.links() assert len(links) == 2 - config = session.emane.get_model_config(node1.id * 1000, EmaneRfPipeModel.name) + config = session.emane.get_model_config(config_id, EmaneRfPipeModel.name) assert config["datarate"] == datarate From c4a724ee10c2c3807a79385f64141baef0497056 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 2 Sep 2020 12:08:21 -0700 Subject: [PATCH 08/36] daemon: added more wrapping classes, updated grpc.clientw to leverage wrapped classes for listened events --- daemon/core/api/grpc/clientw.py | 77 +++++++++++----- daemon/core/api/grpc/wrappers.py | 147 ++++++++++++++++++++++++++++++- 2 files changed, 202 insertions(+), 22 deletions(-) diff --git a/daemon/core/api/grpc/clientw.py b/daemon/core/api/grpc/clientw.py index 10de850b3..25ee8b2f7 100644 --- a/daemon/core/api/grpc/clientw.py +++ b/daemon/core/api/grpc/clientw.py @@ -108,36 +108,65 @@ def create_iface( ) -def stream_listener(stream: Any, handler: Callable[[core_pb2.Event], None]) -> None: +def throughput_listener( + stream: Any, handler: Callable[[wrappers.ThroughputsEvent], None] +) -> None: """ - Listen for stream events and provide them to the handler. + Listen for throughput events and provide them to the handler. + + :param stream: grpc stream that will provide events + :param handler: function that handles an event + :return: nothing + """ + try: + for event_proto in stream: + event = wrappers.ThroughputsEvent.from_proto(event_proto) + handler(event) + except grpc.RpcError as e: + if e.code() == grpc.StatusCode.CANCELLED: + logging.debug("throughput stream closed") + else: + logging.exception("throughput stream error") + + +def cpu_listener( + stream: Any, handler: Callable[[wrappers.CpuUsageEvent], None] +) -> None: + """ + Listen for cpu events and provide them to the handler. :param stream: grpc stream that will provide events :param handler: function that handles an event :return: nothing """ try: - for event in stream: + for event_proto in stream: + event = wrappers.CpuUsageEvent.from_proto(event_proto) handler(event) except grpc.RpcError as e: if e.code() == grpc.StatusCode.CANCELLED: - logging.debug("stream closed") + logging.debug("cpu stream closed") else: - logging.exception("stream error") + logging.exception("cpu stream error") -def start_streamer(stream: Any, handler: Callable[[core_pb2.Event], None]) -> None: +def event_listener(stream: Any, handler: Callable[[wrappers.Event], None]) -> None: """ - Convenience method for starting a grpc stream thread for handling streamed events. + Listen for session events and provide them to the handler. :param stream: grpc stream that will provide events :param handler: function that handles an event :return: nothing """ - thread = threading.Thread( - target=stream_listener, args=(stream, handler), daemon=True - ) - thread.start() + try: + for event_proto in stream: + event = wrappers.Event.from_proto(event_proto) + handler(event) + except grpc.RpcError as e: + if e.code() == grpc.StatusCode.CANCELLED: + logging.debug("session stream closed") + else: + logging.exception("session stream error") class CoreGrpcClient: @@ -469,12 +498,11 @@ def alert( response = self.stub.SessionAlert(request) return response.result - # TODO: determine best path for handling non proto events def events( self, session_id: int, - handler: Callable[[core_pb2.Event], None], - events: List[core_pb2.Event] = None, + handler: Callable[[wrappers.Event], None], + events: List[wrappers.EventType] = None, ) -> grpc.Future: """ Listen for session events. @@ -487,12 +515,14 @@ def events( """ request = core_pb2.EventsRequest(session_id=session_id, events=events) stream = self.stub.Events(request) - start_streamer(stream, handler) + thread = threading.Thread( + target=event_listener, args=(stream, handler), daemon=True + ) + thread.start() return stream - # TODO: determine best path for handling non proto events def throughputs( - self, session_id: int, handler: Callable[[core_pb2.ThroughputsEvent], None] + self, session_id: int, handler: Callable[[wrappers.ThroughputsEvent], None] ) -> grpc.Future: """ Listen for throughput events with information for interfaces and bridges. @@ -504,12 +534,14 @@ def throughputs( """ request = core_pb2.ThroughputsRequest(session_id=session_id) stream = self.stub.Throughputs(request) - start_streamer(stream, handler) + thread = threading.Thread( + target=throughput_listener, args=(stream, handler), daemon=True + ) + thread.start() return stream - # TODO: determine best path for handling non proto events def cpu_usage( - self, delay: int, handler: Callable[[core_pb2.CpuUsageEvent], None] + self, delay: int, handler: Callable[[wrappers.CpuUsageEvent], None] ) -> grpc.Future: """ Listen for cpu usage events with the given repeat delay. @@ -520,7 +552,10 @@ def cpu_usage( """ request = core_pb2.CpuUsageRequest(delay=delay) stream = self.stub.CpuUsage(request) - start_streamer(stream, handler) + thread = threading.Thread( + target=cpu_listener, args=(stream, handler), daemon=True + ) + thread.start() return stream def add_node(self, session_id: int, node: wrappers.Node, source: str = None) -> int: diff --git a/daemon/core/api/grpc/wrappers.py b/daemon/core/api/grpc/wrappers.py index 285bdfce8..3fc087fa4 100644 --- a/daemon/core/api/grpc/wrappers.py +++ b/daemon/core/api/grpc/wrappers.py @@ -1,7 +1,7 @@ from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple from core.api.grpc import ( common_pb2, @@ -100,6 +100,15 @@ class ServiceAction(Enum): VALIDATE = 3 +class EventType: + SESSION = 0 + NODE = 1 + LINK = 2 + CONFIG = 3 + EXCEPTION = 4 + FILE = 5 + + @dataclass class ConfigService: group: str @@ -285,6 +294,15 @@ def from_proto(cls, proto: core_pb2.ThroughputsEvent) -> "ThroughputsEvent": ) +@dataclass +class CpuUsageEvent: + usage: float + + @classmethod + def from_proto(cls, proto: core_pb2.CpuUsageEvent) -> "CpuUsageEvent": + return CpuUsageEvent(usage=proto.usage) + + @dataclass class SessionLocation: x: float @@ -776,6 +794,133 @@ def from_proto(cls, proto: core_pb2.NodeEvent) -> "NodeEvent": ) +@dataclass +class SessionEvent: + node_id: int + event: int + name: str + data: str + time: float + + @classmethod + def from_proto(cls, proto: core_pb2.SessionEvent) -> "SessionEvent": + return SessionEvent( + node_id=proto.node_id, + event=proto.event, + name=proto.name, + data=proto.data, + time=proto.time, + ) + + +@dataclass +class FileEvent: + message_type: MessageType + node_id: int + name: str + mode: str + number: int + type: str + source: str + data: str + compressed_data: str + + @classmethod + def from_proto(cls, proto: core_pb2.FileEvent) -> "FileEvent": + return FileEvent( + message_type=MessageType(proto.message_type), + node_id=proto.node_id, + name=proto.name, + mode=proto.mode, + number=proto.number, + type=proto.type, + source=proto.source, + data=proto.data, + compressed_data=proto.compressed_data, + ) + + +@dataclass +class ConfigEvent: + message_type: MessageType + node_id: int + object: str + type: int + data_types: List[int] + data_values: str + captions: str + bitmap: str + possible_values: str + groups: str + iface_id: int + network_id: int + opaque: str + + @classmethod + def from_proto(cls, proto: core_pb2.ConfigEvent) -> "ConfigEvent": + return ConfigEvent( + message_type=MessageType(proto.message_type), + node_id=proto.node_id, + object=proto.object, + type=proto.type, + data_types=list(proto.data_types), + data_values=proto.data_values, + captions=proto.captions, + bitmap=proto.bitmap, + possible_values=proto.possible_values, + groups=proto.groups, + iface_id=proto.iface_id, + network_id=proto.network_id, + opaque=proto.opaque, + ) + + +@dataclass +class Event: + session_id: int + source: str = None + session_event: SessionEvent = None + node_event: NodeEvent = None + link_event: LinkEvent = None + config_event: Any = None + exception_event: ExceptionEvent = None + file_event: FileEvent = None + + @classmethod + def from_proto(cls, proto: core_pb2.Event) -> "Event": + source = proto.source if proto.source else None + node_event = None + link_event = None + exception_event = None + session_event = None + file_event = None + config_event = None + if proto.HasField("node_event"): + node_event = NodeEvent.from_proto(proto.node_event) + elif proto.HasField("link_event"): + link_event = LinkEvent.from_proto(proto.link_event) + elif proto.HasField("exception_event"): + exception_event = ExceptionEvent.from_proto( + proto.session_id, proto.exception_event + ) + elif proto.HasField("session_event"): + session_event = SessionEvent.from_proto(proto.session_event) + elif proto.HasField("file_event"): + file_event = FileEvent.from_proto(proto.file_event) + elif proto.HasField("config_event"): + config_event = ConfigEvent.from_proto(proto.config_event) + return Event( + session_id=proto.session_id, + source=source, + node_event=node_event, + link_event=link_event, + exception_event=exception_event, + session_event=session_event, + file_event=file_event, + config_event=config_event, + ) + + @dataclass class EmaneEventChannel: group: str From 98a51ce17d09a83eafd06ed99b78e830e0bf1e46 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 5 Sep 2020 10:19:44 -0700 Subject: [PATCH 09/36] grpc: implemented wrapper stream classes for using the wrapped client --- daemon/core/api/grpc/clientw.py | 56 +++++++++++++++++++++++++------- daemon/core/api/grpc/wrappers.py | 42 ++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 12 deletions(-) diff --git a/daemon/core/api/grpc/clientw.py b/daemon/core/api/grpc/clientw.py index 25ee8b2f7..c0f06dc26 100644 --- a/daemon/core/api/grpc/clientw.py +++ b/daemon/core/api/grpc/clientw.py @@ -5,7 +5,8 @@ import logging import threading from contextlib import contextmanager -from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple +from queue import Queue +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple import grpc @@ -30,7 +31,6 @@ from core.api.grpc.core_pb2 import ExecuteScriptRequest from core.api.grpc.emane_pb2 import ( EmaneLinkRequest, - EmanePathlossesRequest, GetEmaneConfigRequest, GetEmaneEventChannelRequest, GetEmaneModelConfigRequest, @@ -69,6 +69,42 @@ from core.emulator.data import IpPrefixes +class MoveNodesStreamer: + def __init__(self) -> None: + self.queue: Queue = Queue() + + def send(self, request: Optional[wrappers.MoveNodesRequest]) -> None: + self.queue.put(request) + + def next(self) -> Optional[core_pb2.MoveNodesRequest]: + request: Optional[wrappers.MoveNodesRequest] = self.queue.get() + if request: + return request.to_proto() + else: + return request + + def iter(self): + return iter(self.next, None) + + +class EmanePathlossesStreamer: + def __init__(self) -> None: + self.queue: Queue = Queue() + + def send(self, request: Optional[wrappers.EmanePathlossesRequest]) -> None: + self.queue.put(request) + + def next(self) -> Optional[emane_pb2.EmanePathlossesRequest]: + request: Optional[wrappers.EmanePathlossesRequest] = self.queue.get() + if request: + return request.to_proto() + else: + return request + + def iter(self): + return iter(self.next, None) + + class InterfaceHelper: """ Convenience class to help generate IP4 and IP6 addresses for gRPC clients. @@ -627,16 +663,15 @@ def edit_node( response = self.stub.EditNode(request) return response.result - # TODO: determine path to stream non proto requests - def move_nodes(self, move_iterator: Iterable[core_pb2.MoveNodesRequest]) -> None: + def move_nodes(self, streamer: MoveNodesStreamer) -> None: """ Stream node movements using the provided iterator. - :param move_iterator: iterator for generating node movements + :param streamer: move nodes streamer :return: nothing :raises grpc.RpcError: when session or nodes do not exist """ - self.stub.MoveNodes(move_iterator) + self.stub.MoveNodes(streamer.iter()) def delete_node(self, session_id: int, node_id: int, source: str = None) -> bool: """ @@ -1396,19 +1431,16 @@ def wlan_link( response = self.stub.WlanLink(request) return response.result - # TODO: determine path to stream non proto requests - def emane_pathlosses( - self, pathloss_iterator: Iterable[EmanePathlossesRequest] - ) -> None: + def emane_pathlosses(self, streamer: EmanePathlossesStreamer) -> None: """ Stream EMANE pathloss events. - :param pathloss_iterator: iterator for sending emane pathloss events + :param streamer: emane pathlosses streamer :return: nothing :raises grpc.RpcError: when a pathloss event session or one of the nodes do not exist """ - self.stub.EmanePathlosses(pathloss_iterator) + self.stub.EmanePathlosses(streamer.iter()) def connect(self) -> None: """ diff --git a/daemon/core/api/grpc/wrappers.py b/daemon/core/api/grpc/wrappers.py index 3fc087fa4..2146198ad 100644 --- a/daemon/core/api/grpc/wrappers.py +++ b/daemon/core/api/grpc/wrappers.py @@ -934,3 +934,45 @@ def from_proto( return EmaneEventChannel( group=proto.group, port=proto.port, device=proto.device ) + + +@dataclass +class EmanePathlossesRequest: + session_id: int + node1_id: int + rx1: float + iface1_id: int + node2_id: int + rx2: float + iface2_id: int + + def to_proto(self) -> emane_pb2.EmanePathlossesRequest: + return emane_pb2.EmanePathlossesRequest( + session_id=self.session_id, + node1_id=self.node1_id, + rx1=self.rx1, + iface1_id=self.iface1_id, + node2_id=self.node2_id, + rx2=self.rx2, + iface2_id=self.iface2_id, + ) + + +@dataclass +class MoveNodesRequest: + session_id: int + node_id: int + source: str = None + position: Position = None + geo: Geo = None + + def to_proto(self) -> core_pb2.MoveNodesRequest: + position = self.position.to_proto() if self.position else None + geo = self.geo.to_proto() if self.geo else None + return core_pb2.MoveNodesRequest( + session_id=self.session_id, + node_id=self.node_id, + source=self.source, + position=position, + geo=geo, + ) From 82d87445b61d5c200e3c58af59a97343997e3367 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Sat, 5 Sep 2020 10:34:08 -0700 Subject: [PATCH 10/36] grpc: added some convenience functions for move node streaming in wrapped client --- daemon/core/api/grpc/clientw.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/daemon/core/api/grpc/clientw.py b/daemon/core/api/grpc/clientw.py index c0f06dc26..36ec69ad4 100644 --- a/daemon/core/api/grpc/clientw.py +++ b/daemon/core/api/grpc/clientw.py @@ -6,7 +6,7 @@ import threading from contextlib import contextmanager from queue import Queue -from typing import Any, Callable, Dict, Generator, List, Optional, Tuple +from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Tuple import grpc @@ -70,12 +70,34 @@ class MoveNodesStreamer: - def __init__(self) -> None: + def __init__(self, session_id: int = None, source: str = None) -> None: + self.session_id = session_id + self.source = source self.queue: Queue = Queue() - def send(self, request: Optional[wrappers.MoveNodesRequest]) -> None: + def send_position(self, node_id: int, x: float, y: float) -> None: + position = wrappers.Position(x=x, y=y) + request = wrappers.MoveNodesRequest( + session_id=self.session_id, + node_id=node_id, + source=self.source, + position=position, + ) + self.send(request) + + def send_geo(self, node_id: int, lon: float, lat: float, alt: float) -> None: + geo = wrappers.Geo(lon=lon, lat=lat, alt=alt) + request = wrappers.MoveNodesRequest( + session_id=self.session_id, node_id=node_id, source=self.source, geo=geo + ) + self.send(request) + + def send(self, request: wrappers.MoveNodesRequest) -> None: self.queue.put(request) + def stop(self) -> None: + self.queue.put(None) + def next(self) -> Optional[core_pb2.MoveNodesRequest]: request: Optional[wrappers.MoveNodesRequest] = self.queue.get() if request: @@ -83,7 +105,7 @@ def next(self) -> Optional[core_pb2.MoveNodesRequest]: else: return request - def iter(self): + def iter(self) -> Iterable: return iter(self.next, None) From d981d88a6ff7b4243f17a21ebcac65454999c805 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Sep 2020 10:27:06 -0700 Subject: [PATCH 11/36] daemon: update how emane is started on nodes, fixing issue with multiple interfaces running emane, added test case to check on this in the future --- daemon/core/emane/commeffect.py | 4 +- daemon/core/emane/emanemanager.py | 136 +++++++++++++++--------------- daemon/core/executables.py | 2 + daemon/core/nodes/base.py | 25 +++++- daemon/core/nodes/interface.py | 5 +- daemon/core/nodes/physical.py | 28 +++++- daemon/core/xml/emanexml.py | 113 +++++++++++++------------ daemon/tests/emane/test_emane.py | 41 +++++++++ 8 files changed, 227 insertions(+), 127 deletions(-) diff --git a/daemon/core/emane/commeffect.py b/daemon/core/emane/commeffect.py index 0fa70a92b..13ec53f77 100644 --- a/daemon/core/emane/commeffect.py +++ b/daemon/core/emane/commeffect.py @@ -80,7 +80,7 @@ def build_xml_files(self, config: Dict[str, str], iface: CoreInterface) -> None: nem_name = emanexml.nem_file_name(iface) shim_name = emanexml.shim_file_name(iface) etree.SubElement(nem_element, "shim", definition=shim_name) - emanexml.create_iface_file(iface, nem_element, "nem", nem_name) + emanexml.create_node_file(iface.node, nem_element, "nem", nem_name) # create and write shim document shim_element = etree.Element( @@ -99,7 +99,7 @@ def build_xml_files(self, config: Dict[str, str], iface: CoreInterface) -> None: ff = config["filterfile"] if ff.strip() != "": emanexml.add_param(shim_element, "filterfile", ff) - emanexml.create_iface_file(iface, shim_element, "shim", shim_name) + emanexml.create_node_file(iface.node, shim_element, "shim", shim_name) # create transport xml emanexml.create_transport_xml(iface, config) diff --git a/daemon/core/emane/emanemanager.py b/daemon/core/emane/emanemanager.py index a6b299279..de4c37bca 100644 --- a/daemon/core/emane/emanemanager.py +++ b/daemon/core/emane/emanemanager.py @@ -6,6 +6,7 @@ import os import threading from collections import OrderedDict +from dataclasses import dataclass, field from enum import Enum from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Type @@ -28,7 +29,7 @@ RegisterTlvs, ) from core.errors import CoreCommandError, CoreError -from core.nodes.base import CoreNode, NodeBase +from core.nodes.base import CoreNetworkBase, CoreNode, CoreNodeBase, NodeBase from core.nodes.interface import CoreInterface, TunTap from core.xml import emanexml @@ -68,6 +69,13 @@ class EmaneState(Enum): NOT_READY = 2 +@dataclass +class StartData: + emane_net: EmaneNet + node: CoreNodeBase + ifaces: List[CoreInterface] = field(default_factory=list) + + class EmaneManager(ModelManager): """ EMANE controller object. Lives in a Session instance and is used for @@ -126,20 +134,15 @@ def get_iface_config( self, emane_net: EmaneNet, iface: CoreInterface ) -> Dict[str, str]: """ - Retrieve configuration for a given interface. + Retrieve configuration for a given interface, first checking for interface + specific config, node specific config, network specific config, and finally + falling back to the default configuration settings. :param emane_net: emane network the interface is connected to :param iface: interface running emane :return: net, node, or interface model configuration """ model_name = emane_net.model.name - # don"t use default values when interface config is the same as net - # note here that using iface.node.id as key allows for only one type - # of each model per node; - # TODO: use both node and interface as key - # Adamson change: first check for iface config keyed by "node:iface.name" - # (so that nodes w/ multiple interfaces of same conftype can have - # different configs for each separate interface) config = None # try to retrieve interface specific configuration if iface.node_id is not None: @@ -345,36 +348,45 @@ def startup(self) -> EmaneState: self.buildeventservicexml() with self._emane_node_lock: logging.info("emane building xmls...") - for node_id in sorted(self._emane_nets): - emane_net = self._emane_nets[node_id] - if not emane_net.model: - logging.error("emane net(%s) has no model", emane_net.name) - continue - for iface in emane_net.get_ifaces(): - self.start_iface(emane_net, iface) + start_data = self.get_start_data() + for data in start_data: + self.start_node(data) if self.links_enabled(): self.link_monitor.start() return EmaneState.SUCCESS - def start_iface(self, emane_net: EmaneNet, iface: CoreInterface) -> None: - if not iface.node: - logging.error( - "emane net(%s) connected interface(%s) missing node", - emane_net.name, - iface.name, - ) - return + def get_start_data(self) -> List[StartData]: + node_map = {} + for node_id in sorted(self._emane_nets): + emane_net = self._emane_nets[node_id] + if not emane_net.model: + logging.error("emane net(%s) has no model", emane_net.name) + continue + for iface in emane_net.get_ifaces(): + if not iface.node: + logging.error( + "emane net(%s) connected interface(%s) missing node", + emane_net.name, + iface.name, + ) + continue + start_node = node_map.setdefault( + iface.node, StartData(emane_net, iface.node) + ) + start_node.ifaces.append(iface) + start_nodes = sorted(node_map.values(), key=lambda x: x.node.id) + for start_node in start_nodes: + start_node.ifaces = sorted(start_node.ifaces, key=lambda x: x.node_id) + return start_nodes + + def start_node(self, data: StartData) -> None: control_net = self.session.add_remove_control_net( 0, remove=False, conf_required=False ) - nem_id = self.next_nem_id() - self.set_nem(nem_id, iface) - self.write_nem(iface, nem_id) - emanexml.build_platform_xml(self, control_net, emane_net, iface, nem_id) - config = self.get_iface_config(emane_net, iface) - emane_net.model.build_xml_files(config, iface) - self.start_daemon(iface) - self.install_iface(emane_net, iface) + emanexml.build_platform_xml(self, control_net, data) + self.start_daemon(data.node) + for iface in data.ifaces: + self.install_iface(data.emane_net, iface) def set_nem(self, nem_id: int, iface: CoreInterface) -> None: if nem_id in self.nems_to_ifaces: @@ -435,8 +447,21 @@ def shutdown(self) -> None: logging.info("stopping EMANE daemons") if self.links_enabled(): self.link_monitor.stop() - self.deinstall_ifaces() - self.stopdaemons() + # shutdown interfaces and stop daemons + kill_emaned = "killall -q emane" + start_data = self.get_start_data() + for data in start_data: + node = data.node + if not node.up: + continue + for iface in data.ifaces: + if isinstance(node, CoreNode): + iface.shutdown() + iface.poshook = None + if isinstance(node, CoreNode): + node.cmd(kill_emaned, wait=False) + else: + node.host_cmd(kill_emaned, wait=False) self.stopeventmonitor() def check_node_models(self) -> None: @@ -524,7 +549,7 @@ def buildeventservicexml(self) -> None: ) ) - def start_daemon(self, iface: CoreInterface) -> None: + def start_daemon(self, node: CoreNodeBase) -> None: """ Start one EMANE daemon per node having a radio. Add a control network even if the user has not configured one. @@ -539,8 +564,7 @@ def start_daemon(self, iface: CoreInterface) -> None: emanecmd = f"emane -d -l {loglevel}" if realtime: emanecmd += " -r" - node = iface.node - if iface.is_virtual(): + if isinstance(node, CoreNode): otagroup, _otaport = self.get_config("otamanagergroup").split(":") otadev = self.get_config("otamanagerdevice") otanetidx = self.session.get_control_net_index(otadev) @@ -569,35 +593,19 @@ def start_daemon(self, iface: CoreInterface) -> None: if eventservicenetidx >= 0 and eventgroup != otagroup: node.node_net_client.create_route(eventgroup, eventdev) # start emane - log_file = os.path.join(node.nodedir, f"{iface.name}-emane.log") - platform_xml = os.path.join(node.nodedir, f"{iface.name}-platform.xml") + log_file = os.path.join(node.nodedir, f"{node.name}-emane.log") + platform_xml = os.path.join(node.nodedir, f"{node.name}-platform.xml") args = f"{emanecmd} -f {log_file} {platform_xml}" node.cmd(args) logging.info("node(%s) emane daemon running: %s", node.name, args) else: path = self.session.session_dir - log_file = os.path.join(path, f"{iface.name}-emane.log") - platform_xml = os.path.join(path, f"{iface.name}-platform.xml") + log_file = os.path.join(path, f"{node.name}-emane.log") + platform_xml = os.path.join(path, f"{node.name}-platform.xml") emanecmd += f" -f {log_file} {platform_xml}" node.host_cmd(emanecmd, cwd=path) logging.info("node(%s) host emane daemon running: %s", node.name, emanecmd) - def stopdaemons(self) -> None: - """ - Kill the appropriate EMANE daemons. - """ - kill_emaned = "killall -q emane" - for node_id in sorted(self._emane_nets): - emane_net = self._emane_nets[node_id] - for iface in emane_net.get_ifaces(): - node = iface.node - if not node.up: - continue - if iface.is_raw(): - node.host_cmd(kill_emaned, wait=False) - else: - node.cmd(kill_emaned, wait=False) - def install_iface(self, emane_net: EmaneNet, iface: CoreInterface) -> None: config = self.get_iface_config(emane_net, iface) external = config.get("external", "0") @@ -609,17 +617,6 @@ def install_iface(self, emane_net: EmaneNet, iface: CoreInterface) -> None: iface.poshook = emane_net.setnemposition iface.setposition() - def deinstall_ifaces(self) -> None: - """ - Uninstall TUN/TAP virtual interfaces. - """ - for key in sorted(self._emane_nets): - emane_net = self._emane_nets[key] - for iface in emane_net.get_ifaces(): - if iface.is_virtual(): - iface.shutdown() - iface.poshook = None - def doeventmonitor(self) -> bool: """ Returns boolean whether or not EMANE events will be monitored. @@ -783,6 +780,9 @@ def handlelocationeventtoxyz( self.session.broadcast_node(node) return True + def is_emane_net(self, net: Optional[CoreNetworkBase]) -> bool: + return isinstance(net, EmaneNet) + def emanerunning(self, node: CoreNode) -> bool: """ Return True if an EMANE process associated with the given node is running, diff --git a/daemon/core/executables.py b/daemon/core/executables.py index 7b7f80b71..16f159fce 100644 --- a/daemon/core/executables.py +++ b/daemon/core/executables.py @@ -11,6 +11,7 @@ MOUNT: str = "mount" UMOUNT: str = "umount" OVS_VSCTL: str = "ovs-vsctl" +TEST: str = "test" COMMON_REQUIREMENTS: List[str] = [ BASH, @@ -21,6 +22,7 @@ SYSCTL, TC, UMOUNT, + TEST, ] VCMD_REQUIREMENTS: List[str] = [VNODED, VCMD] OVS_REQUIREMENTS: List[str] = [OVS_VSCTL] diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 4cf6ea8d1..cf0c87388 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -16,7 +16,7 @@ from core.emulator.data import InterfaceData, LinkData, LinkOptions from core.emulator.enumerations import LinkTypes, MessageFlags, NodeTypes from core.errors import CoreCommandError, CoreError -from core.executables import MOUNT, VNODED +from core.executables import MOUNT, TEST, VNODED from core.nodes.client import VnodeClient from core.nodes.interface import CoreInterface, TunTap, Veth from core.nodes.netclient import LinuxNetClient, get_net_client @@ -294,6 +294,16 @@ def new_iface( """ raise NotImplementedError + @abc.abstractmethod + def path_exists(self, path: str) -> bool: + """ + Determines if a file or directory path exists. + + :param path: path to file or directory + :return: True if path exists, False otherwise + """ + raise NotImplementedError + def add_config_service(self, service_class: "ConfigServiceType") -> None: """ Adds a configuration service to the node. @@ -602,6 +612,19 @@ def cmd(self, args: str, wait: bool = True, shell: bool = False) -> str: args = self.client.create_cmd(args, shell) return self.server.remote_cmd(args, wait=wait) + def path_exists(self, path: str) -> bool: + """ + Determines if a file or directory path exists. + + :param path: path to file or directory + :return: True if path exists, False otherwise + """ + try: + self.cmd(f"{TEST} -e {path}") + return True + except CoreCommandError: + return False + def termcmdstring(self, sh: str = "/bin/sh") -> str: """ Create a terminal command string. diff --git a/daemon/core/nodes/interface.py b/daemon/core/nodes/interface.py index 20dc8fd31..bc242eacb 100644 --- a/daemon/core/nodes/interface.py +++ b/daemon/core/nodes/interface.py @@ -528,8 +528,9 @@ def nodedevexists(): # check if this is an EMANE interface; if so, continue # waiting if EMANE is still running should_retry = count < 5 - is_emane_running = self.node.session.emane.emanerunning(self.node) - if all([should_retry, self.net.is_emane, is_emane_running]): + is_emane = self.session.emane.is_emane_net(self.net) + is_emane_running = self.session.emane.emanerunning(self.node) + if all([should_retry, is_emane, is_emane_running]): count += 1 else: raise RuntimeError("node device failed to exist") diff --git a/daemon/core/nodes/physical.py b/daemon/core/nodes/physical.py index f48a0d10d..22819f6db 100644 --- a/daemon/core/nodes/physical.py +++ b/daemon/core/nodes/physical.py @@ -11,7 +11,7 @@ from core.emulator.distributed import DistributedServer from core.emulator.enumerations import NodeTypes, TransportType from core.errors import CoreCommandError, CoreError -from core.executables import MOUNT, UMOUNT +from core.executables import MOUNT, TEST, UMOUNT from core.nodes.base import CoreNetworkBase, CoreNodeBase from core.nodes.interface import CoreInterface from core.nodes.network import CoreNetwork, GreTap @@ -55,6 +55,19 @@ def shutdown(self) -> None: self.rmnodedir() + def path_exists(self, path: str) -> bool: + """ + Determines if a file or directory path exists. + + :param path: path to file or directory + :return: True if path exists, False otherwise + """ + try: + self.host_cmd(f"{TEST} -e {path}") + return True + except CoreCommandError: + return False + def termcmdstring(self, sh: str = "/bin/sh") -> str: """ Create a terminal command string. @@ -291,6 +304,19 @@ def shutdown(self) -> None: self.up = False self.restorestate() + def path_exists(self, path: str) -> bool: + """ + Determines if a file or directory path exists. + + :param path: path to file or directory + :return: True if path exists, False otherwise + """ + try: + self.host_cmd(f"{TEST} -e {path}") + return True + except CoreCommandError: + return False + def new_iface( self, net: CoreNetworkBase, iface_data: InterfaceData ) -> CoreInterface: diff --git a/daemon/core/xml/emanexml.py b/daemon/core/xml/emanexml.py index 88aeaa97b..7ccf2c4b1 100644 --- a/daemon/core/xml/emanexml.py +++ b/daemon/core/xml/emanexml.py @@ -7,15 +7,15 @@ from core import utils from core.config import Configuration -from core.emane.nodes import EmaneNet from core.emulator.distributed import DistributedServer from core.errors import CoreError +from core.nodes.base import CoreNode, CoreNodeBase from core.nodes.interface import CoreInterface from core.nodes.network import CtrlNet from core.xml import corexml if TYPE_CHECKING: - from core.emane.emanemanager import EmaneManager + from core.emane.emanemanager import EmaneManager, StartData from core.emane.emanemodel import EmaneModel _MAC_PREFIX = "02:02" @@ -78,23 +78,22 @@ def create_file( corexml.write_xml_file(xml_element, file_path, doctype=doctype) -def create_iface_file( - iface: CoreInterface, xml_element: etree.Element, doc_name: str, file_name: str +def create_node_file( + node: CoreNodeBase, xml_element: etree.Element, doc_name: str, file_name: str ) -> None: """ Create emane xml for an interface. - :param iface: interface running emane + :param node: node running emane :param xml_element: root element to write to file :param doc_name: name to use in the emane doctype :param file_name: name of xml file :return: """ - node = iface.node - if iface.is_raw(): - file_path = os.path.join(node.session.session_dir, file_name) - else: + if isinstance(node, CoreNode): file_path = os.path.join(node.nodedir, file_name) + else: + file_path = os.path.join(node.session.session_dir, file_name) create_file(xml_element, doc_name, file_path, node.server) @@ -143,11 +142,7 @@ def add_configurations( def build_platform_xml( - emane_manager: "EmaneManager", - control_net: CtrlNet, - emane_net: EmaneNet, - iface: CoreInterface, - nem_id: int, + emane_manager: "EmaneManager", control_net: CtrlNet, data: "StartData" ) -> None: """ Create platform xml for a specific node. @@ -156,50 +151,62 @@ def build_platform_xml( configurations :param control_net: control net node for this emane network - :param emane_net: emane network associated with interface - :param iface: interface running emane - :param nem_id: nem id to use for this interface + :param data: start data for a node connected to emane and associated interfaces :return: the next nem id that can be used for creating platform xml files """ - # build nem xml - nem_definition = nem_file_name(iface) - nem_element = etree.Element( - "nem", id=str(nem_id), name=iface.localname, definition=nem_definition - ) - - # check if this is an external transport, get default config if an interface - # specific one does not exist - config = emane_manager.get_iface_config(emane_net, iface) - if is_external(config): - nem_element.set("transport", "external") - platform_endpoint = "platformendpoint" - add_param(nem_element, platform_endpoint, config[platform_endpoint]) - transport_endpoint = "transportendpoint" - add_param(nem_element, transport_endpoint, config[transport_endpoint]) - else: - transport_name = transport_file_name(iface) - transport_element = etree.SubElement( - nem_element, "transport", definition=transport_name - ) - add_param(transport_element, "device", iface.name) - + # create top level platform element transport_configs = {"otamanagerdevice", "eventservicedevice"} platform_element = etree.Element("platform") for configuration in emane_manager.emane_config.emulator_config: name = configuration.id - if iface.is_raw() and name in transport_configs: + if not isinstance(data.node, CoreNode) and name in transport_configs: value = control_net.brname else: value = emane_manager.get_config(name) add_param(platform_element, name, value) - platform_element.append(nem_element) - mac = _MAC_PREFIX + ":00:00:" - mac += f"{(nem_id >> 8) & 0xFF:02X}:{nem_id & 0xFF:02X}" - iface.set_mac(mac) + + # create nem xml entries for all interfaces + emane_net = data.emane_net + for iface in data.ifaces: + nem_id = emane_manager.next_nem_id() + emane_manager.set_nem(nem_id, iface) + emane_manager.write_nem(iface, nem_id) + config = emane_manager.get_iface_config(emane_net, iface) + emane_net.model.build_xml_files(config, iface) + + # build nem xml + nem_definition = nem_file_name(iface) + nem_element = etree.Element( + "nem", id=str(nem_id), name=iface.localname, definition=nem_definition + ) + + # check if this is an external transport, get default config if an interface + # specific one does not exist + config = emane_manager.get_iface_config(emane_net, iface) + if is_external(config): + nem_element.set("transport", "external") + platform_endpoint = "platformendpoint" + add_param(nem_element, platform_endpoint, config[platform_endpoint]) + transport_endpoint = "transportendpoint" + add_param(nem_element, transport_endpoint, config[transport_endpoint]) + else: + transport_name = transport_file_name(iface) + transport_element = etree.SubElement( + nem_element, "transport", definition=transport_name + ) + add_param(transport_element, "device", iface.name) + + # add nem element to platform element + platform_element.append(nem_element) + + # generate and assign interface mac address based on nem id + mac = _MAC_PREFIX + ":00:00:" + mac += f"{(nem_id >> 8) & 0xFF:02X}:{nem_id & 0xFF:02X}" + iface.set_mac(mac) doc_name = "platform" - file_name = f"{iface.name}-platform.xml" - create_iface_file(iface, platform_element, doc_name, file_name) + file_name = f"{data.node.name}-platform.xml" + create_node_file(data.node, platform_element, doc_name, file_name) def create_transport_xml(iface: CoreInterface, config: Dict[str, str]) -> None: @@ -220,16 +227,16 @@ def create_transport_xml(iface: CoreInterface, config: Dict[str, str]) -> None: # get emane model cnfiguration flowcontrol = config.get("flowcontrolenable", "0") == "1" - if iface.is_virtual(): + if isinstance(iface.node, CoreNode): device_path = "/dev/net/tun_flowctl" - if not os.path.exists(device_path): + if not iface.node.path_exists(device_path): device_path = "/dev/net/tun" add_param(transport_element, "devicepath", device_path) if flowcontrol: add_param(transport_element, "flowcontrolenable", "on") doc_name = "transport" transport_name = transport_file_name(iface) - create_iface_file(iface, transport_element, doc_name, transport_name) + create_node_file(iface.node, transport_element, doc_name, transport_name) def create_phy_xml( @@ -250,7 +257,7 @@ def create_phy_xml( phy_element, emane_model.phy_config, config, emane_model.config_ignore ) file_name = phy_file_name(iface) - create_iface_file(iface, phy_element, "phy", file_name) + create_node_file(iface.node, phy_element, "phy", file_name) def create_mac_xml( @@ -273,7 +280,7 @@ def create_mac_xml( mac_element, emane_model.mac_config, config, emane_model.config_ignore ) file_name = mac_file_name(iface) - create_iface_file(iface, mac_element, "mac", file_name) + create_node_file(iface.node, mac_element, "mac", file_name) def create_nem_xml( @@ -298,7 +305,7 @@ def create_nem_xml( phy_name = phy_file_name(iface) etree.SubElement(nem_element, "phy", definition=phy_name) nem_name = nem_file_name(iface) - create_iface_file(iface, nem_element, "nem", nem_name) + create_node_file(iface.node, nem_element, "nem", nem_name) def create_event_service_xml( @@ -351,7 +358,7 @@ def nem_file_name(iface: CoreInterface) -> str: :param iface: interface running emane :return: nem xm file name """ - append = "-raw" if iface.is_raw() else "" + append = "-raw" if not isinstance(iface.node, CoreNode) else "" return f"{iface.name}-nem{append}.xml" diff --git a/daemon/tests/emane/test_emane.py b/daemon/tests/emane/test_emane.py index 6c21e4d7b..ccbfb4461 100644 --- a/daemon/tests/emane/test_emane.py +++ b/daemon/tests/emane/test_emane.py @@ -44,6 +44,47 @@ def ping( class TestEmane: + def test_two_emane_interfaces(self, session: Session): + """ + Test nodes running multiple emane interfaces. + + :param core.emulator.coreemu.EmuSession session: session for test + """ + # create emane node for networking the core nodes + session.set_location(47.57917, -122.13232, 2.00000, 1.0) + options = NodeOptions() + options.set_position(80, 50) + options.emane = EmaneIeee80211abgModel.name + emane_net1 = session.add_node(EmaneNet, options=options) + options.emane = EmaneRfPipeModel.name + emane_net2 = session.add_node(EmaneNet, options=options) + + # create nodes + options = NodeOptions(model="mdr") + options.set_position(150, 150) + node1 = session.add_node(CoreNode, options=options) + options.set_position(300, 150) + node2 = session.add_node(CoreNode, options=options) + + # create interfaces + ip_prefix1 = IpPrefixes("10.0.0.0/24") + ip_prefix2 = IpPrefixes("10.0.1.0/24") + for i, node in enumerate([node1, node2]): + node.setposition(x=150 * (i + 1), y=150) + iface_data = ip_prefix1.create_iface(node) + session.add_link(node.id, emane_net1.id, iface1_data=iface_data) + iface_data = ip_prefix2.create_iface(node) + session.add_link(node.id, emane_net2.id, iface1_data=iface_data) + + # instantiate session + session.instantiate() + + # ping node2 from node1 on both interfaces and check success + status = ping(node1, node2, ip_prefix1, count=5) + assert not status + status = ping(node1, node2, ip_prefix2, count=5) + assert not status + @pytest.mark.parametrize("model", _EMANE_MODELS) def test_models( self, session: Session, model: Type[EmaneModel], ip_prefixes: IpPrefixes From 9e3d6681c5e17a918d111c2351bd5838c6a25258 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Sep 2020 14:23:37 -0700 Subject: [PATCH 12/36] daemon: added ospf-mdr checkout out to the current latest commit as a sub module to help avoid future changes breaking a working state and a common place to keep files for uninstall --- .gitmodules | 3 +++ ospf-mdr | 1 + tasks.py | 9 +++------ 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 .gitmodules create mode 160000 ospf-mdr diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..ecd2bf1cb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ospf-mdr"] + path = ospf-mdr + url = https://github.com/USNavalResearchLaboratory/ospf-mdr.git diff --git a/ospf-mdr b/ospf-mdr new file mode 160000 index 000000000..26fe5a440 --- /dev/null +++ b/ospf-mdr @@ -0,0 +1 @@ +Subproject commit 26fe5a4401a26760c553fcadfde5311199e89450 diff --git a/tasks.py b/tasks.py index ab73ccc86..716d8f471 100644 --- a/tasks.py +++ b/tasks.py @@ -186,12 +186,9 @@ def install_ospf_mdr(c: Context, os_info: OsInfo, hide: bool) -> None: c.run("sudo apt install -y libtool gawk libreadline-dev git", hide=hide) elif os_info.like == OsLike.REDHAT: c.run("sudo yum install -y libtool gawk readline-devel git", hide=hide) - clone_dir = "/tmp/ospf-mdr" - c.run( - f"git clone https://github.com/USNavalResearchLaboratory/ospf-mdr {clone_dir}", - hide=hide - ) - with c.cd(clone_dir): + ospf_mdr_dir = "ospf-mdr" + c.run(f"git submodule update --init -- {ospf_mdr_dir}", hide=hide) + with c.cd(ospf_mdr_dir): c.run("./bootstrap.sh", hide=hide) c.run( "./configure --disable-doc --enable-user=root --enable-group=root " From 57e6df51d3685aa3c7e580b690e389e850a72ab3 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Sep 2020 16:17:29 -0700 Subject: [PATCH 13/36] daemon: removed ospf-mdr as git submodule, due to conflicts with being a sub directory of an existing automake project, ospf-mdr will now be checked out to parent directory of core --- .gitmodules | 3 --- ospf-mdr | 1 - tasks.py | 8 +++++--- 3 files changed, 5 insertions(+), 7 deletions(-) delete mode 100644 .gitmodules delete mode 160000 ospf-mdr diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index ecd2bf1cb..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "ospf-mdr"] - path = ospf-mdr - url = https://github.com/USNavalResearchLaboratory/ospf-mdr.git diff --git a/ospf-mdr b/ospf-mdr deleted file mode 160000 index 26fe5a440..000000000 --- a/ospf-mdr +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 26fe5a4401a26760c553fcadfde5311199e89450 diff --git a/tasks.py b/tasks.py index 716d8f471..dd86cdf94 100644 --- a/tasks.py +++ b/tasks.py @@ -186,9 +186,11 @@ def install_ospf_mdr(c: Context, os_info: OsInfo, hide: bool) -> None: c.run("sudo apt install -y libtool gawk libreadline-dev git", hide=hide) elif os_info.like == OsLike.REDHAT: c.run("sudo yum install -y libtool gawk readline-devel git", hide=hide) - ospf_mdr_dir = "ospf-mdr" - c.run(f"git submodule update --init -- {ospf_mdr_dir}", hide=hide) - with c.cd(ospf_mdr_dir): + ospf_dir = "../ospf-mdr" + ospf_url = "https://github.com/USNavalResearchLaboratory/ospf-mdr.git" + c.run(f"git clone {ospf_url} {ospf_dir}", hide=hide) + c.run("git checkout 26fe5a4401a26760c553fcadfde5311199e89450", hide=hide) + with c.cd(ospf_dir): c.run("./bootstrap.sh", hide=hide) c.run( "./configure --disable-doc --enable-user=root --enable-group=root " From 202e681fffefa4c8af4e8e258a7206b19d419d1a Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Sep 2020 16:37:23 -0700 Subject: [PATCH 14/36] install: fix ospf mdr repo checkout location --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index dd86cdf94..444ba8905 100644 --- a/tasks.py +++ b/tasks.py @@ -189,8 +189,8 @@ def install_ospf_mdr(c: Context, os_info: OsInfo, hide: bool) -> None: ospf_dir = "../ospf-mdr" ospf_url = "https://github.com/USNavalResearchLaboratory/ospf-mdr.git" c.run(f"git clone {ospf_url} {ospf_dir}", hide=hide) - c.run("git checkout 26fe5a4401a26760c553fcadfde5311199e89450", hide=hide) with c.cd(ospf_dir): + c.run("git checkout 26fe5a4401a26760c553fcadfde5311199e89450", hide=hide) c.run("./bootstrap.sh", hide=hide) c.run( "./configure --disable-doc --enable-user=root --enable-group=root " From 646c6b8f0192b0f8af95ec8eaa673141f0934169 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Sep 2020 16:52:35 -0700 Subject: [PATCH 15/36] install: updated emane install to checkout a specific version to avoid unexpected errors and checkout the repo to parent directory of the core repo --- tasks.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tasks.py b/tasks.py index 444ba8905..1325aa2cc 100644 --- a/tasks.py +++ b/tasks.py @@ -14,6 +14,8 @@ DAEMON_DIR: str = "daemon" DEFAULT_PREFIX: str = "/usr/local" +EMANE_CHECKOUT: str = "v1.2.5" +OSPFMDR_CHECKOUT: str = "26fe5a4401a26760c553fcadfde5311199e89450" class Progress: @@ -190,7 +192,7 @@ def install_ospf_mdr(c: Context, os_info: OsInfo, hide: bool) -> None: ospf_url = "https://github.com/USNavalResearchLaboratory/ospf-mdr.git" c.run(f"git clone {ospf_url} {ospf_dir}", hide=hide) with c.cd(ospf_dir): - c.run("git checkout 26fe5a4401a26760c553fcadfde5311199e89450", hide=hide) + c.run(f"git checkout {OSPFMDR_CHECKOUT}", hide=hide) c.run("./bootstrap.sh", hide=hide) c.run( "./configure --disable-doc --enable-user=root --enable-group=root " @@ -339,7 +341,7 @@ def install_emane(c, verbose=False): p = Progress(verbose) hide = not verbose os_info = get_os() - emane_dir = "/tmp/emane" + emane_dir = "../emane" with p.start("installing system dependencies"): if os_info.like == OsLike.DEBIAN: c.run( @@ -357,13 +359,12 @@ def install_emane(c, verbose=False): "protobuf-devel python3-setuptools", hide=hide, ) + emane_url = "https://github.com/adjacentlink/emane.git" with p.start("cloning emane"): - c.run( - f"git clone https://github.com/adjacentlink/emane.git {emane_dir}", - hide=hide - ) + c.run(f"git clone {emane_url} {emane_dir}", hide=hide) with p.start("building emane"): with c.cd(emane_dir): + c.run(f"git checkout {EMANE_CHECKOUT}", hide=hide) c.run("./autogen.sh", hide=hide) c.run("PYTHON=python3 ./configure --prefix=/usr", hide=hide) c.run("make -j$(nproc)", hide=hide) From 8234a5ed7e6c4beb8416a8855a4fce68c61568b9 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Sep 2020 17:49:05 -0700 Subject: [PATCH 16/36] install: adjust emane repo path to use an absolute path with installing from poetry --- tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks.py b/tasks.py index 1325aa2cc..0a3408fbe 100644 --- a/tasks.py +++ b/tasks.py @@ -341,7 +341,6 @@ def install_emane(c, verbose=False): p = Progress(verbose) hide = not verbose os_info = get_os() - emane_dir = "../emane" with p.start("installing system dependencies"): if os_info.like == OsLike.DEBIAN: c.run( @@ -359,6 +358,7 @@ def install_emane(c, verbose=False): "protobuf-devel python3-setuptools", hide=hide, ) + emane_dir = Path("../emane") emane_url = "https://github.com/adjacentlink/emane.git" with p.start("cloning emane"): c.run(f"git clone {emane_url} {emane_dir}", hide=hide) @@ -371,9 +371,10 @@ def install_emane(c, verbose=False): with p.start("installing emane"): with c.cd(emane_dir): c.run("sudo make install", hide=hide) + emane_python_dir = emane_dir.joinpath("src/python") with p.start("installing python binding for core"): with c.cd(DAEMON_DIR): - c.run(f"poetry run pip install {emane_dir}/src/python", hide=hide) + c.run(f"poetry run pip install {emane_python_dir.absolute()}", hide=hide) @task( From b9a14fbe0cfc3b116ef164f3449d6fb7ed28d817 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Wed, 9 Sep 2020 18:42:26 -0700 Subject: [PATCH 17/36] install: fixed invoke cd not supporting pathlib.Path --- tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks.py b/tasks.py index 0a3408fbe..0a0f685b1 100644 --- a/tasks.py +++ b/tasks.py @@ -358,7 +358,8 @@ def install_emane(c, verbose=False): "protobuf-devel python3-setuptools", hide=hide, ) - emane_dir = Path("../emane") + emane_dir = "../emane" + emane_python_dir = Path(emane_dir).joinpath("src/python") emane_url = "https://github.com/adjacentlink/emane.git" with p.start("cloning emane"): c.run(f"git clone {emane_url} {emane_dir}", hide=hide) @@ -371,7 +372,6 @@ def install_emane(c, verbose=False): with p.start("installing emane"): with c.cd(emane_dir): c.run("sudo make install", hide=hide) - emane_python_dir = emane_dir.joinpath("src/python") with p.start("installing python binding for core"): with c.cd(DAEMON_DIR): c.run(f"poetry run pip install {emane_python_dir.absolute()}", hide=hide) From 0668d0a49bb8832216669fc823ea5f6391e3a7b5 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Sep 2020 15:05:49 -0700 Subject: [PATCH 18/36] install: add option to support building a wheel from poetry and installing locally --- daemon/pyproject.toml | 9 ++++++- install.sh | 9 +++++-- tasks.py | 55 +++++++++++++++++++++++++------------------ 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/daemon/pyproject.toml b/daemon/pyproject.toml index 55bfabe40..2bea0b898 100644 --- a/daemon/pyproject.toml +++ b/daemon/pyproject.toml @@ -6,7 +6,14 @@ authors = ["Boeing Research and Technology"] license = "BSD-2-Clause" repository = "https://github.com/coreemu/core" documentation = "https://coreemu.github.io/core/" -include = ["core/gui/data/**/*", "core/configservices/*/templates"] +include = [ + "core/api/grpc/*", + "core/configservices/*/templates", + "core/constants.py", + "core/gui/data/**/*", +] +exclude = ["core/constants.py.in"] + [tool.poetry.dependencies] python = "^3.6" diff --git a/install.sh b/install.sh index 5e5a9b117..59584afa1 100755 --- a/install.sh +++ b/install.sh @@ -14,7 +14,8 @@ fi dev="" verbose="" prefix="" -while getopts "dvp:" opt; do +local="" +while getopts "dvlp:" opt; do case ${opt} in d) dev="-d" @@ -22,6 +23,9 @@ while getopts "dvp:" opt; do v) verbose="-v" ;; + l) + local="-l" + ;; p) prefix="-p ${OPTARG}" ;; @@ -30,6 +34,7 @@ while getopts "dvp:" opt; do echo "" >&2 echo "-v enable verbose install" >&2 echo "-d enable developer install" >&2 + echo "-l enable local install, not compatible with developer install" >&2 echo "-p install prefix, defaults to /usr/local" >&2 exit 1 ;; @@ -54,4 +59,4 @@ python3 -m pip install --user pipx python3 -m pipx ensurepath export PATH=$PATH:~/.local/bin pipx install invoke -inv install ${dev} ${verbose} ${prefix} +inv install ${dev} ${verbose} ${local} ${prefix} diff --git a/tasks.py b/tasks.py index 0a0f685b1..71bab859e 100644 --- a/tasks.py +++ b/tasks.py @@ -171,13 +171,18 @@ def install_core(c: Context, hide: bool) -> None: c.run("sudo make install", hide=hide) -def install_poetry(c: Context, dev: bool, hide: bool) -> None: +def install_poetry(c: Context, dev: bool, local: bool, hide: bool) -> None: c.run("pipx install poetry", hide=hide) - args = "" if dev else "--no-dev" - with c.cd(DAEMON_DIR): - c.run(f"poetry install {args}", hide=hide) - if dev: - c.run("poetry run pre-commit install", hide=hide) + if local: + with c.cd(DAEMON_DIR): + c.run("poetry build -f wheel", hide=hide) + c.run("python3 -m pip install dist/*") + else: + args = "" if dev else "--no-dev" + with c.cd(DAEMON_DIR): + c.run(f"poetry install {args}", hide=hide) + if dev: + c.run("poetry run pre-commit install", hide=hide) def install_ospf_mdr(c: Context, os_info: OsInfo, hide: bool) -> None: @@ -243,10 +248,11 @@ def install_service(c, verbose=False, prefix=DEFAULT_PREFIX): @task( help={ "verbose": "enable verbose", - "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}" + "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}", + "local": "determines if core will install to local system, default is False", }, ) -def install_scripts(c, verbose=False, prefix=DEFAULT_PREFIX): +def install_scripts(c, local=False, verbose=False, prefix=DEFAULT_PREFIX): """ install core script files, modified to leverage virtual environment """ @@ -259,7 +265,7 @@ def install_scripts(c, verbose=False, prefix=DEFAULT_PREFIX): lines = f.readlines() first = lines[0].strip() # modify python scripts to point to virtual environment - if first == "#!/usr/bin/env python3": + if not local and first == "#!/usr/bin/env python3": lines[0] = f"#!{python}\n" temp = NamedTemporaryFile("w", delete=False) for line in lines: @@ -273,16 +279,17 @@ def install_scripts(c, verbose=False, prefix=DEFAULT_PREFIX): c.run(f"sudo cp {script} {dest}", hide=hide) # setup core python helper - core_python = bin_dir.joinpath("core-python") - temp = NamedTemporaryFile("w", delete=False) - temp.writelines([ - "#!/bin/bash\n", - f'exec "{python}" "$@"\n', - ]) - temp.close() - c.run(f"sudo cp {temp.name} {core_python}", hide=hide) - c.run(f"sudo chmod 755 {core_python}", hide=hide) - os.unlink(temp.name) + if not local: + core_python = bin_dir.joinpath("core-python") + temp = NamedTemporaryFile("w", delete=False) + temp.writelines([ + "#!/bin/bash\n", + f'exec "{python}" "$@"\n', + ]) + temp.close() + c.run(f"sudo cp {temp.name} {core_python}", hide=hide) + c.run(f"sudo chmod 755 {core_python}", hide=hide) + os.unlink(temp.name) # install core configuration file config_dir = "/etc/core" @@ -295,13 +302,15 @@ def install_scripts(c, verbose=False, prefix=DEFAULT_PREFIX): help={ "dev": "install development mode", "verbose": "enable verbose", - "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}" + "local": "determines if core will install to local system, default is False", + "prefix": f"prefix where scripts are installed, default is {DEFAULT_PREFIX}", }, ) -def install(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX): +def install(c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX): """ install core, poetry, scripts, service, and ospf mdr """ + print(f"installing core locally: {local}") print(f"installing core with prefix: {prefix}") c.run("sudo -v", hide=True) p = Progress(verbose) @@ -318,9 +327,9 @@ def install(c, dev=False, verbose=False, prefix=DEFAULT_PREFIX): with p.start("installing vcmd/gui"): install_core(c, hide) with p.start("installing poetry virtual environment"): - install_poetry(c, dev, hide) + install_poetry(c, dev, local, hide) with p.start("installing scripts and /etc/core"): - install_scripts(c, hide, prefix) + install_scripts(c, local, hide, prefix) with p.start("installing systemd service"): install_service(c, hide, prefix) with p.start("installing ospf mdr"): From f83f71c074173b3414256e6f3f4d346c65c5a1f1 Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Sep 2020 15:16:28 -0700 Subject: [PATCH 19/36] install: use sudo when installing wheel for non-user specific install --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 71bab859e..17493c17c 100644 --- a/tasks.py +++ b/tasks.py @@ -176,7 +176,7 @@ def install_poetry(c: Context, dev: bool, local: bool, hide: bool) -> None: if local: with c.cd(DAEMON_DIR): c.run("poetry build -f wheel", hide=hide) - c.run("python3 -m pip install dist/*") + c.run("sudo python3 -m pip install dist/*") else: args = "" if dev else "--no-dev" with c.cd(DAEMON_DIR): From efdce20afb2c9a766b44339b23fd83cba04a84ff Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Sep 2020 15:51:01 -0700 Subject: [PATCH 20/36] install: add local install support for the install emane task --- tasks.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tasks.py b/tasks.py index 17493c17c..845e0f166 100644 --- a/tasks.py +++ b/tasks.py @@ -340,9 +340,10 @@ def install(c, dev=False, verbose=False, local=False, prefix=DEFAULT_PREFIX): @task( help={ "verbose": "enable verbose", + "local": "used determine if core is installed locally, default is False", }, ) -def install_emane(c, verbose=False): +def install_emane(c, verbose=False, local=False): """ install emane and the python bindings """ @@ -382,8 +383,14 @@ def install_emane(c, verbose=False): with c.cd(emane_dir): c.run("sudo make install", hide=hide) with p.start("installing python binding for core"): - with c.cd(DAEMON_DIR): - c.run(f"poetry run pip install {emane_python_dir.absolute()}", hide=hide) + if local: + with c.cd(str(emane_python_dir)): + c.run("sudo python3 -m pip install .", hide=hide) + else: + with c.cd(DAEMON_DIR): + c.run( + f"poetry run pip install {emane_python_dir.absolute()}", hide=hide + ) @task( From 7790d4aa00bc3e22c5f55b68ac2e258db9b6000d Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 11 Sep 2020 16:26:13 -0700 Subject: [PATCH 21/36] docs: update install.md to denote the option for installing locally and reference that as a possibility in other instructions --- docs/install.md | 72 ++++++++++++++++++++++++++++++++++++------------- install.sh | 2 +- tasks.py | 7 +++-- 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/docs/install.md b/docs/install.md index 7b5014c40..021939944 100644 --- a/docs/install.md +++ b/docs/install.md @@ -74,11 +74,14 @@ sudo apt remove core ## Automated Installation -The automated install will install the various tools needed to help automate -the CORE installation (python3, pip, pipx, invoke, poetry). The script will -also automatically clone, build, and install the latest version of OSPF MDR. -Finally it will install CORE scripts and a systemd service, which have -been modified to use the installed poetry created virtual environment. +The automated install will install do the following: +* install base tools needed for installation + * python3, pip, pipx, invoke, poetry +* installs system dependencies for building core +* installs latest version of [OPSF MDR](https://github.com/USNavalResearchLaboratory/ospf-mdr) +* installs core into poetry managed virtual environment or locally, if flag is passed +* installs scripts pointing to python interpreter being used +* installs systemd service, disabled by default After installation has completed you should be able to run the various CORE scripts for running core. @@ -92,10 +95,11 @@ git clone https://github.com/coreemu/core.git cd core # run install script -# script usage: install.sh [-d] [-v] +# script usage: install.sh [-v] [-d] [-l] [-p ] # # -v enable verbose install # -d enable developer install +# -l enable local install, not compatible with developer install # -p install prefix, defaults to /usr/local ./install.sh ``` @@ -117,16 +121,17 @@ After the installation complete it will have installed the following scripts. | Name | Description | |---|---| +| core-cleanup | tool to help removed lingering core created containers, bridges, directories | +| core-cli | tool to query, open xml files, and send commands using gRPC | | core-daemon | runs the backed core server providing TLV and gRPC APIs | | core-gui | runs the legacy tcl/tk based GUI | -| core-pygui | runs the new python/tk based GUI | -| core-cleanup | tool to help removed lingering core created containers, bridges, directories | | core-imn-to-xml | tool to help automate converting a .imn file to .xml format | +| core-manage | tool to add, remove, or check for services, models, and node types | +| core-pygui | runs the new python/tk based GUI | +| core-python | provides a convenience for running the core python virtual environment | | core-route-monitor | tool to help monitor traffic across nodes and feed that to SDT | | core-service-update | tool to update automate modifying a legacy service to match current naming | | coresendmsg | tool to send TLV API commands from command line | -| core-cli | tool to query, open xml files, and send commands using gRPC | -| core-manage | tool to add, remove, or check for services, models, and node types | ## Running User Scripts @@ -142,28 +147,57 @@ environment interpreter or to run a script within it. core-python