From b7b3a17ceced225601959102a1f077a93bb88034 Mon Sep 17 00:00:00 2001 From: Noguchi Ko <92015366+noguchiko@users.noreply.github.com> Date: Thu, 12 May 2022 16:47:40 +0900 Subject: [PATCH 1/6] yang: add goldstone-telemetry This change adds a new goldstone primitive model goldstone-telemetry. The model defines configuration parameters for telemetry subscriptions and a notification for telemetry updates. The model allows a software serves it to manage subscriptions and send telemetry updates. --- yang/goldstone-telemetry.yang | 295 ++++++++++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 yang/goldstone-telemetry.yang diff --git a/yang/goldstone-telemetry.yang b/yang/goldstone-telemetry.yang new file mode 100644 index 00000000..ed502ab1 --- /dev/null +++ b/yang/goldstone-telemetry.yang @@ -0,0 +1,295 @@ +module goldstone-telemetry { + + yang-version "1.1"; + + namespace "http://goldstone.net/yang/goldstone-telemetry"; + prefix gs-telemetry; + + organization + "Goldstone"; + + description + "This module contains a collection of YANG definitions for + managing telemetry subscriptions. The data schema is inspired by + gNMI subscribe request. + + subscribe request: A request from a telemetry collector to + subscribe streaming telemetries. It may have multiple + subscriptions. + subscription: A unit of telemetry notifications. It is tied to + a specific data tree path. + + See also: + - https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md + - https://github.com/openconfig/gnmi/blob/master/proto/gnmi/gnmi.proto + "; + + revision 2022-05-25 { + description + "Initial version."; + reference + "0.1.0"; + } + + grouping subscription-config { + description + "Configuration parameters relating to the subscription."; + + leaf id { + type uint32; + description + "User defined identifier of the subscription."; + } + + leaf path { + type string; + description + "Path to the data tree node to subscribe."; + } + + leaf mode { + type enumeration { + enum TARGET_DEFINED { + description + "A notification provider selects the relevant mode for + each node. ON_CHANGE or SAMPLE."; + } + enum ON_CHANGE { + description + "Notifications are sent on value change."; + } + enum SAMPLE { + description + "Notifications are sent at sample interval."; + } + } + description + "Mode of the subscription."; + } + + leaf sample-interval { + type uint64; + units nanoseconds; + description + "Sampling interval for SAMPLE mode in nanoseconds."; + } + + leaf suppress-redundant { + type boolean; + description + "Notifications for leaf nodes which value has not changed + since the last notification are not sent. Notifications are + sent for those individual leaf nodes in the subscription that + have changed. It is an optional parameter for SAMPLE mode."; + } + + leaf heartbeat-interval { + type uint64; + units nanoseconds; + description + "Maximum allowable silent period in nanoseconds. If the mode + is ON_CHANGE, a notification will be sent once per heartbeat + interval regardless of whether the value has changed or not. + If the mode is SAMPLE, a notification will be sent per + heartbeat interval regardless of whether the + suppress-redundant is set to true. The value 0 means + heartbeat updates are disabled."; + } + } + + grouping subscription-state { + description + "Operational state data relating to the subscription."; + } + + grouping subscription-top { + description + "Top level grouping for subscription configuration and + operational state data."; + + container subscriptions { + description + "Top level container for subscriptions."; + + list subscription { + key "id"; + description + "List of subscribe requests."; + + leaf id { + type leafref { + path "../config/id"; + } + description + "Reference to the identifier of the subscription."; + } + + container config { + description + "Configuration parameters of the subscription."; + uses subscription-config; + } + + container state { + config false; + description + "Operational stetes of the subscription."; + uses subscription-config; + uses subscription-state; + } + } + } + } + + grouping subscribe-request-config { + description + "Configuration parameters relating to the subscribe request."; + + leaf id { + type uint32; + description + "User defined identifier of the subscribe request."; + } + + leaf mode { + type enumeration { + enum STREAM { + description + "Notifications are streamed."; + } + enum ONCE { + description + "Notifications are sent once-off."; + } + enum POLL { + description + "Notifications are sent as response to a polling + request."; + } + } + description + "Mode of the subscribe request."; + } + + leaf updates-only { + type boolean; + description + "Send only updates to current state. The initial state is not + sent. If mode is ONCE or POLL, notifications will never be + sent."; + } + } + + grouping subscribe-request-state { + description + "Operational state data relating to the subscribe request."; + } + + grouping telemetry-top { + description + "Top level grouping for telemetry configuration and operational + state data."; + + container subscribe-requests { + description + "Top level container for subscribe requests."; + + list subscribe-request { + key "id"; + description + "List of subscribe requests."; + + leaf id { + type leafref { + path "../config/id"; + } + description + "Reference to the identifier of the subscribe request."; + } + + container config { + description + "Configuration parameters of the subscribe request."; + uses subscribe-request-config; + } + + container state { + config false; + description + "Operational stetes of the subscribe request."; + uses subscribe-request-config; + uses subscribe-request-state; + } + + uses subscription-top; + } + } + } + + uses telemetry-top; + + rpc poll { + description + "Polling request for a subscribe request. This will trigger a + polled update process if the mode is POLL."; + input { + leaf id { + type leafref { + path + "/gs-telemetry:subscribe-requests" + + "/gs-telemetry:subscribe-request" + + "/gs-telemetry:id"; + } + } + } + } + + notification telemetry-notify-event { + description + "Telemetry notification."; + + leaf type { + type enumeration { + enum UPDATE { + description + "The data tree node is created or updated."; + } + enum DELETE { + description + "The data tree node is deleted."; + } + enum SYNC_RESPONSE { + description + "Indicates that all data values have been transmitted at least + once."; + } + } + } + + leaf request-id { + type uint32; + description + "Reference to the identifier of the subscribe request."; + } + + leaf subscription-id { + type uint32; + description + "Reference to the identifier of the subscription in the + subscribe request."; + } + + leaf path { + type string; + description + "Path to the data tree node of the notification."; + } + + leaf json-data { + type string; + description + "Value of the node in json string."; + } + } +} + From 3d771c2ec805f58abce0f55c8f5a203cb629b147 Mon Sep 17 00:00:00 2001 From: Noguchi Ko <92015366+noguchiko@users.noreply.github.com> Date: Tue, 17 May 2022 16:56:57 +0900 Subject: [PATCH 2/6] system/telemetry: add streaming telemetry application This change adds an application for the streaming telemetry feature. It allows users to manage telemetry subscriptions and receive notifications about telemetry updates via the goldstone-telemetry model. --- Makefile | 20 +- README.md | 8 +- docker/agent.Dockerfile | 8 + k8s/prep.yaml | 2 +- k8s/system-telemetry.yaml | 81 + scripts/gs-yang.py | 3 + src/system/telemetry/README.md | 38 + .../goldstone/system/telemetry/__init__.py | 0 .../goldstone/system/telemetry/main.py | 76 + .../goldstone/system/telemetry/path.py | 134 ++ .../goldstone/system/telemetry/store.py | 226 ++ .../goldstone/system/telemetry/telemetry.py | 630 ++++++ src/system/telemetry/requirements.txt | 1 + src/system/telemetry/setup.py | 20 + src/system/telemetry/tests/__init__.py | 0 src/system/telemetry/tests/test.py | 12 + src/system/telemetry/tests/test_path.py | 57 + src/system/telemetry/tests/test_store.py | 162 ++ src/system/telemetry/tests/test_telemetry.py | 1809 +++++++++++++++++ 19 files changed, 3279 insertions(+), 8 deletions(-) create mode 100644 k8s/system-telemetry.yaml create mode 100644 src/system/telemetry/README.md create mode 100644 src/system/telemetry/goldstone/system/telemetry/__init__.py create mode 100644 src/system/telemetry/goldstone/system/telemetry/main.py create mode 100644 src/system/telemetry/goldstone/system/telemetry/path.py create mode 100644 src/system/telemetry/goldstone/system/telemetry/store.py create mode 100644 src/system/telemetry/goldstone/system/telemetry/telemetry.py create mode 100644 src/system/telemetry/requirements.txt create mode 100644 src/system/telemetry/setup.py create mode 100644 src/system/telemetry/tests/__init__.py create mode 100644 src/system/telemetry/tests/test.py create mode 100644 src/system/telemetry/tests/test_path.py create mode 100644 src/system/telemetry/tests/test_store.py create mode 100644 src/system/telemetry/tests/test_telemetry.py diff --git a/Makefile b/Makefile index acce4be5..8e4ab261 100644 --- a/Makefile +++ b/Makefile @@ -37,11 +37,12 @@ TRANSPONDER_YANG ?= ./yang/goldstone-transponder.yang all: builder images tester host-packages -images: south-images north-images xlate-images +images: south-images north-images xlate-images system-images GS_SOUTH_AGENTS ?= south-sonic south-tai south-onlp south-system south-gearbox south-dpll south-netlink GS_NORTH_AGENTS ?= north-cli north-snmp north-netconf north-notif north-gnmi GS_XLATE_AGENTS ?= xlate-oc +GS_SYSTEM_AGENTS ?= system-telemetry south-images: $(GS_SOUTH_AGENTS) @@ -49,7 +50,9 @@ north-images: $(GS_NORTH_AGENTS) xlate-images: $(GS_XLATE_AGENTS) -$(GS_SOUTH_AGENTS) $(GS_NORTH_AGENTS) $(GS_XLATE_AGENTS): +system-images: $(GS_SYSTEM_AGENTS) + +$(GS_SOUTH_AGENTS) $(GS_NORTH_AGENTS) $(GS_XLATE_AGENTS) $(GS_SYSTEM_AGENTS): $(call build_agent_image,$@) north-snmp: snmpd @@ -90,11 +93,11 @@ cmd: lint: which black && exit `black -q --diff --exclude "src/north/snmp/src|src/north/gnmi/goldstone/north/gnmi/proto" src | wc -l` TRANSPONDER_YANG=/tmp/test.yang $(MAKE) yang && diff /tmp/test.yang $(TRANSPONDER_YANG) - scripts/gs-yang.py --lint south-sonic south-onlp south-tai south-system xlate-oc --search-dirs yang sm/openconfig - scripts/gs-yang.py --lint south-gearbox south-onlp south-tai south-system xlate-oc --search-dirs yang sm/openconfig + scripts/gs-yang.py --lint south-sonic south-onlp south-tai south-system xlate-oc system-telemetry --search-dirs yang sm/openconfig + scripts/gs-yang.py --lint south-gearbox south-onlp south-tai south-system xlate-oc system-telemetry --search-dirs yang sm/openconfig grep -rnI 'print(' src || exit 0 && exit 1 -unittest: unittest-lib unittest-cli unittest-gearbox unittest-dpll unittest-openconfig unittest-tai unittest-sonic unittest-gnmi +unittest: unittest-lib unittest-cli unittest-gearbox unittest-dpll unittest-openconfig unittest-tai unittest-sonic unittest-gnmi unittest-telemetry rust-unittest: unittest-netlink @@ -137,6 +140,11 @@ unittest-openconfig: scripts/gs-yang.py --install xlate-oc south-onlp south-tai south-gearbox south-system --search-dirs yang sm/openconfig cd src/xlate/openconfig && PYTHONPATH=../../lib python -m unittest -v -f $(TEST_CASE) +unittest-telemetry: + $(MAKE) clean-sysrepo + scripts/gs-yang.py --install system-telemetry south-gearbox --search-dirs yang sm/openconfig + cd src/system/telemetry && PYTHONPATH=../../lib python -m unittest -v -f $(TEST_CASE) + unittest-tai: $(MAKE) clean-sysrepo scripts/gs-yang.py --install south-tai --search-dirs yang @@ -159,4 +167,4 @@ unittest-gnmi: cd src/north/gnmi && make proto scripts/gs-yang.py --install xlate-oc --search-dirs yang sm/openconfig cd src/north/gnmi && PYTHONPATH=../../lib python -m unittest -v -f $(TEST_CASE) - cd src/north/gnmi && make clean \ No newline at end of file + cd src/north/gnmi && make clean diff --git a/README.md b/README.md index eb58baa5..c9595037 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ The components in this repo are pre-installed in Goldstone NOS. - e.g) Switch ASIC, Transponder, Gearbox, Peripheral devices(Thermal sensors, LED, fan etc..) - Goldstone management translation daemons that translate standarized YANG models and Goldstone YANG models (Red boxes in the following diagram) - OpenConfig Translator +- Goldstone management system daemons that provide service to north daemons by only interacting with south or xlate daemons + - Streaming telemetry Server - Goldstone YANG models - The schemas that are used between north and south daemons @@ -85,7 +87,7 @@ The intention to have native YANG models is to fully cover what the underneath h Using the standard YANG models ([OpenConfig](https://www.openconfig.net/), [OpenROADM](http://openroadm.org/) etc..) is also supported by using translater daemons. -`goldstone-mgmt` framework has three kinds of daemon which interact with sysrepo datastore. +`goldstone-mgmt` framework has four kinds of daemon which interact with sysrepo datastore. - north daemon - provides northbound API (CLI, NETCONF, SNMP, RESTCONF, gNMI etc..) @@ -97,6 +99,10 @@ Using the standard YANG models ([OpenConfig](https://www.openconfig.net/), [Open - translation daemon - translator of the standarized YANG models and Goldstone YANG models - source code under [`src/xlate`](https://github.com/oopt-goldstone/goldstone-mgmt/tree/master/src/xlate) +- system daemon + - provides system utility services for north daemons + - optionally uses native YANG models to interact with sysrepo + - source code under [`src/system`](https://github.com/oopt-goldstone/goldstone-mgmt/tree/master/src/system) ### How to build diff --git a/docker/agent.Dockerfile b/docker/agent.Dockerfile index 7fa517d9..634f1182 100644 --- a/docker/agent.Dockerfile +++ b/docker/agent.Dockerfile @@ -278,6 +278,14 @@ RUN --mount=type=bind,source=src/xlate/openconfig,target=/src,rw pip install /sr RUN mkdir -p /current RUN --mount=type=bind,source=scripts/,target=/scripts,rw cp /scripts/operational-modes.json /current +#--- +# system-telemetry +#--- + +FROM base AS system-telemetry + +RUN --mount=type=bind,source=src/system/telemetry,target=/src,rw pip install /src + #--- # default image #--- diff --git a/k8s/prep.yaml b/k8s/prep.yaml index d8aa8659..fe38b45a 100644 --- a/k8s/prep.yaml +++ b/k8s/prep.yaml @@ -47,7 +47,7 @@ spec: - name: prep-sysrepo image: ghcr.io/oopt-goldstone/mgmt/north-cli:latest imagePullPolicy: IfNotPresent - command: ["gs-yang.py", "--install", "south-onlp", "south-sonic", "south-tai", "south-system", "xlate-oc"] + command: ["gs-yang.py", "--install", "south-onlp", "south-sonic", "south-tai", "south-system", "xlate-oc", "system-telemetry"] volumeMounts: - name: shm mountPath: /dev/shm diff --git a/k8s/system-telemetry.yaml b/k8s/system-telemetry.yaml new file mode 100644 index 00000000..73ac2f2f --- /dev/null +++ b/k8s/system-telemetry.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: system-telemetry + labels: + gs-mgmt: system-telemetry + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-telemetry + labels: + gs-mgmt: system-telemetry +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view +subjects: +- kind: ServiceAccount + name: system-telemetry + namespace: default + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: system-telemetry + labels: + app: gs-mgmt + gs-mgmt: system-telemetry +spec: + selector: + matchLabels: + app: system-telemetry + template: + metadata: + labels: + app: system-telemetry + spec: + serviceAccountName: system-telemetry + initContainers: + - name: wait-prep + image: docker.io/lachlanevenson/k8s-kubectl:latest + imagePullPolicy: IfNotPresent + command: ['kubectl', 'wait', '--for=condition=complete', 'job/prep-gs-mgmt'] + containers: + - name: system-telemetry + image: ghcr.io/oopt-goldstone/mgmt/system-telemetry:latest + imagePullPolicy: IfNotPresent + command: ['gssystemd-telemetry'] + args: ['--verbose'] + volumeMounts: + - name: shm + mountPath: /dev/shm + - name: sysrepo + mountPath: /var/lib/sysrepo + livenessProbe: + httpGet: + path: /healthz + port: liveness-port + failureThreshold: 10 + periodSeconds: 5 + timeoutSeconds: 5 + startupProbe: + httpGet: + path: /healthz + port: liveness-port + failureThreshold: 30 + periodSeconds: 10 + ports: + - name: liveness-port + containerPort: 8080 + volumes: + - name: shm + hostPath: + path: /dev/shm + - name: sysrepo + hostPath: + path: /var/lib/sysrepo diff --git a/scripts/gs-yang.py b/scripts/gs-yang.py index c088a437..2dc168f8 100755 --- a/scripts/gs-yang.py +++ b/scripts/gs-yang.py @@ -53,6 +53,9 @@ "optical-transport/openconfig-transport-line-common", "optical-transport/openconfig-transport-types", ], + "system-telemetry": [ + "goldstone-telemetry", + ], } DEFAULT_YANG_DIR = "/var/lib/goldstone/yang" diff --git a/src/system/telemetry/README.md b/src/system/telemetry/README.md new file mode 100644 index 00000000..cd95cfac --- /dev/null +++ b/src/system/telemetry/README.md @@ -0,0 +1,38 @@ +# Streaming Telemetry Server + +The streaming telemetry server provides `goldstone-telemetry` service. It allows north daemons to subscribe configuration/operational state changes of the device. + +## Supported models and revisions + +- goldstone-telemetry 2022-05-25 + +## Prerequisites + +- Python >= 3.8 +- Goldstone patched sysrepo-python +- Goldstone patched libyang-python + +Other required python packages are listed in `requirements.txt`. + +## Install + +```sh +sudo pip3 install . +``` + +## Usage + +```sh +$ gssystemd-telemetry -h +usage: gssystemd-telemetry [-h] [-v] + +options: + -h, --help show this help message and exit + -v, --verbose enable detailed output +``` + +Example: + +```sh +gssystemd-telemetry +``` diff --git a/src/system/telemetry/goldstone/system/telemetry/__init__.py b/src/system/telemetry/goldstone/system/telemetry/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/system/telemetry/goldstone/system/telemetry/main.py b/src/system/telemetry/goldstone/system/telemetry/main.py new file mode 100644 index 00000000..b8bec394 --- /dev/null +++ b/src/system/telemetry/goldstone/system/telemetry/main.py @@ -0,0 +1,76 @@ +"""main() function for the streaming telemetry server.""" + + +import logging +import asyncio +import argparse +import signal +import itertools +from goldstone.lib.util import start_probe, call +from goldstone.lib.connector.sysrepo import Connector +from .store import InMemorySubscriptionStore, InMemoryTelemetryStore +from .telemetry import TelemetryServer + + +logger = logging.getLogger(__name__) + + +def main(): + async def _main(): + loop = asyncio.get_event_loop() + stop_event = asyncio.Event() + loop.add_signal_handler(signal.SIGINT, stop_event.set) + loop.add_signal_handler(signal.SIGTERM, stop_event.set) + + conn = Connector() + subscription_store = InMemorySubscriptionStore() + telemetry_store = InMemoryTelemetryStore() + gsserver = TelemetryServer(conn, subscription_store, telemetry_store) + servers = [gsserver] + + try: + tasks = list( + itertools.chain.from_iterable([await s.start() for s in servers]) + ) + + runner = await start_probe("/healthz", "0.0.0.0", 8080) + tasks.append(stop_event.wait()) + done, pending = await asyncio.wait( + tasks, return_when=asyncio.FIRST_COMPLETED + ) + logger.debug("done: %s, pending: %s", done, pending) + for task in done: + e = task.exception() + if e: + raise e + finally: + if runner: + await runner.cleanup() + for s in servers: + await call(s.stop) + conn.stop() + + parser = argparse.ArgumentParser() + parser.add_argument( + "-v", "--verbose", action="store_true", help="enable detailed output" + ) + args = parser.parse_args() + + fmt = "%(levelname)s %(module)s %(funcName)s l.%(lineno)d | %(message)s" + if args.verbose: + logging.basicConfig(level=logging.DEBUG, format=fmt) + for noisy in [ + "hpack", + "kubernetes.client.rest", + "kubernetes_asyncio.client.rest", + ]: + l = logging.getLogger(noisy) + l.setLevel(logging.INFO) + else: + logging.basicConfig(level=logging.INFO, format=fmt) + + asyncio.run(_main()) + + +if __name__ == "__main__": + main() diff --git a/src/system/telemetry/goldstone/system/telemetry/path.py b/src/system/telemetry/goldstone/system/telemetry/path.py new file mode 100644 index 00000000..6469b654 --- /dev/null +++ b/src/system/telemetry/goldstone/system/telemetry/path.py @@ -0,0 +1,134 @@ +"""Path manipulation utilities.""" + + +import logging +import re +import libyang + + +logger = logging.getLogger(__name__) + + +class PathParser: + """A path parser.""" + + REGEX_PTN_LIST_KEY = re.compile(r"\[.*.*\]") + + def __init__(self, ctx): + self._ctx = ctx + + def _is_container(self, data): + return isinstance(data, dict) + + def _is_container_list(self, data): + if isinstance(data, list): + for elem in data: + if isinstance(elem, dict): + return True + return False + + def _find_head_node(self, path): + return self._ctx.find_path(path) + + def _next_node(self, node, target_name): + for child in list(node.children()): + if child.name() == target_name: + return child + + def _remove_list_keys(self, path): + return re.sub(self.REGEX_PTN_LIST_KEY, "", path) + + def _find_node(self, path): + path = self._remove_list_keys(path) + path_elems = path.split("/")[1:] + node = next(self._find_head_node("/" + path_elems[0])) + for node_name in path_elems[1:]: + node_name = node_name.split(":")[-1] + node = self._next_node(node, node_name) + return node + + def _get_list_keys(self, path): + node = self._find_node(path) + keys = [] + for key in node.keys(): + keys.append(key.name()) + return keys + + def _path_with_keys(self, container, path): + keys_str = "" + keys = self._get_list_keys(path) + for key in keys: + val = container[key] + keys_str = f"{keys_str}[{key}='{val}']" + return f"{path}{keys_str}" + + def _get_leaves(self, data, path, leaves): + if self._is_container(data): + for next_node, next_data in data.items(): + next_path = f"{path}/{next_node}" + self._get_leaves(next_data, next_path, leaves) + elif self._is_container_list(data): + for container in data: + next_path = self._path_with_keys(container, path) + self._get_leaves(container, next_path, leaves) + else: + leaves[path] = data + + def _get_path_elems(self, path): + return self._remove_list_keys(path).split("/")[1:] + + def _prune_leaves(self, leaves, path): + path_elems = self._get_path_elems(path) + sub_paths_to_delete = [] + for sub_path in leaves: + sub_path_elems = self._get_path_elems(sub_path) + if len(sub_path_elems) < len(path_elems): + sub_paths_to_delete.append(sub_path) + continue + for index in range(len(path_elems)): + if path_elems[index] != sub_path_elems[index]: + sub_paths_to_delete.append(sub_path) + break + for sub_path in sub_paths_to_delete: + try: + del leaves[sub_path] + except KeyError: + continue + + def parse_dict_into_leaves(self, data, path): + """Parse a data tree dictionaly into path to leaves. + + Args: + data (dict): Data tree in dictionaly to parse. + path (str): Path to the target node. It will be used to prune unnecessary leaves. + + Returns: + dict: Parsed data. + key: Path to a leaf node. + value: Data of a leaf node. + """ + dict_to_parse = {} + top_prefix = self._get_path_elems(path)[0].split(":")[0] + for key, value in data.items(): + new_key = top_prefix + ":" + key + dict_to_parse[new_key] = value + leaves = {} + self._get_leaves(dict_to_parse, "", leaves) + self._prune_leaves(leaves, path) + return leaves + + def is_valid_path(self, path): + """Validate a schema path. + + Args: + path (str): Path to validate. + + Returns: + bool: True for a valid path. False for a invalid path. + """ + try: + if self._find_node(path) is None: + return False + except libyang.LibyangError: + return False + return True diff --git a/src/system/telemetry/goldstone/system/telemetry/store.py b/src/system/telemetry/goldstone/system/telemetry/store.py new file mode 100644 index 00000000..d3ff3f78 --- /dev/null +++ b/src/system/telemetry/goldstone/system/telemetry/store.py @@ -0,0 +1,226 @@ +"""Datastore implementations.""" + + +from abc import abstractmethod +from datetime import datetime +import logging + + +logger = logging.getLogger(__name__) + + +class TelemetryNotExistError(Exception): + pass + + +class TelemetryStore: + """Base class for telemetry datastore. + + Users should depend on this interface instead of subclass implementations. + """ + + @abstractmethod + def set(self, ids, path, value): + """Set a telemetry data. + + If the telemetry data entry does not exist, it creates an entry. + + Args: + ids (tupple of int): Identifier of the subscription + 0: Outer ID. Request ID. + 1: Inner ID. Subscription ID. + path (str): Identifier of the telemetry data. Path to a leaf node. + value (any): The telemetry data to set. + """ + pass + + @abstractmethod + def delete(self, ids, path): + """Delete a telemetry data. + + Args: + ids (tupple of int): Identifier of the subscription + 0: Outer ID. Request ID. + 1: Inner ID. Subscription ID. + path (str): Identifier of the telemetry data. Path to a leaf node. + + Raises: + TelemetryNotExistError: The telemetry data is not found. + """ + pass + + @abstractmethod + def get(self, ids, path): + """Get a telemetry data. + + Args: + ids (tupple of int): Identifier of the subscription + 0: Outer ID. Request ID. + 1: Inner ID. Subscription ID. + path (str): Identifier of the telemetry data. Path to a leaf node. + + Returns: + dict: The telemetry data of the path. + "value" (any): The telemetry data. + "update-time" (datetime): Last update time. + + Raises: + TelemetryNotExistError: The telemetry data is not found. + """ + pass + + @abstractmethod + def list(self, ids): + """Get a telemetry data. + + Args: + ids (tupple of int): Identifier of the subscription + 0: Outer ID. Request ID. + 1: Inner ID. Subscription ID. + + Returns: + list of str: Identifiers of telemetry data. Paths to leaf nodes. + """ + pass + + +class InMemoryTelemetryStore(TelemetryStore): + """A telemetry datastore implementation using volatile memory. + + If you want to keep telemetry data after rebooting your application, you should not use this.""" + + def __init__(self): + self._data = {} + + def set(self, ids, path, value): + outer_id, inner_id = ids + if outer_id not in self._data.keys(): + self._data[outer_id] = {} + if inner_id not in self._data[outer_id].keys(): + self._data[outer_id][inner_id] = {} + data = { + "value": value, + "update-time": datetime.now(), + } + self._data[outer_id][inner_id][path] = data + + def delete(self, ids, path): + outer_id, inner_id = ids + try: + del self._data[outer_id][inner_id][path] + if len(self._data[outer_id][inner_id]) <= 0: + del self._data[outer_id][inner_id] + if len(self._data[outer_id]) <= 0: + del self._data[outer_id] + except KeyError as e: + raise TelemetryNotExistError() from e + + def get(self, ids, path): + outer_id, inner_id = ids + try: + return self._data[outer_id][inner_id][path] + except KeyError as e: + raise TelemetryNotExistError() from e + + def list(self, ids): + outer_id, inner_id = ids + outer = self._data.get(outer_id) + if outer is None: + return [] + inner = outer.get(inner_id) + if inner is None: + return [] + return inner.keys() + + +class SubscriptionExistError(Exception): + pass + + +class SubscriptionNotExistError(Exception): + pass + + +class SubscriptionStore: + """Base class for subscription datastore. + + Users should depend on this interface instead of subclass implementations. + """ + + @abstractmethod + def add(self, id_, subscription): + """Add a subscription. + + Args: + id_ (int): Identifier of the subscription to add. + subscription (Subscription): A subscription to add. + + Raises: + SubscriptionExistError: The subscription has been added. + """ + pass + + @abstractmethod + def delete(self, id_): + """Delete a subscription. + + Args: + id_ (int): Identifier of the subscription to delete. + + Raises: + SubscriptionNotExistError: The subscription is not found. + """ + pass + + @abstractmethod + def get(self, id_): + """Get a subscription. + + Args: + id_ (int): Identifier of the subscription to get. + + Returns: + Subscription: The subscription. + + Raises: + SubscriptionNotExistError: The subscription is not found. + """ + pass + + @abstractmethod + def list(self): + """Get a list of subscription identifiers. + + Returns: + list of str: The list of subscription identifiers. + """ + pass + + +class InMemorySubscriptionStore(SubscriptionStore): + """A subscription datastore implementation using volatile memory. + + If you want to keep subscriptions after rebooting your application, you should not use this.""" + + def __init__(self): + self._subscriptions = {} + + def add(self, id_, subscription): + if id_ in self._subscriptions.keys(): + raise SubscriptionExistError() + self._subscriptions[id_] = subscription + + def delete(self, id_): + try: + del self._subscriptions[id_] + except KeyError as e: + raise SubscriptionNotExistError() from e + + def get(self, id_): + try: + return self._subscriptions[id_] + except KeyError as e: + raise SubscriptionNotExistError() from e + + def list(self): + return list(self._subscriptions.keys()) diff --git a/src/system/telemetry/goldstone/system/telemetry/telemetry.py b/src/system/telemetry/goldstone/system/telemetry/telemetry.py new file mode 100644 index 00000000..6569bc24 --- /dev/null +++ b/src/system/telemetry/goldstone/system/telemetry/telemetry.py @@ -0,0 +1,630 @@ +"""Streaming telemetry servers.""" + + +import logging +import asyncio +import json +from datetime import datetime, timedelta +import sysrepo +import libyang +from goldstone.lib.core import ServerBase, ChangeHandler +from .store import SubscriptionNotExistError, TelemetryNotExistError +from .path import PathParser + + +logger = logging.getLogger(__name__) + + +class ValidationFailedError(Exception): + def __init__(self, msg): + super().__init__() + self.msg = msg + + +class Subscription: + """Base class of subscriptions. + + It retrieves state data and sends notifications with the central datastore. + + Its behavior follows the gNMI specification. See: + https://github.com/openconfig/reference/blob/master/rpc/gnmi/gnmi-specification.md#3515-creating-subscriptions + + Args: + conn (SysrepoConnection): Connection with the central datastore. + config (dict): Configuration data of the subscription. + store (store.TelemetryStore): Datastore for telemetry data. + update_interval (int): Telemetry data update interval in nanoseconds. + """ + + NOTIF_PATH = "goldstone-telemetry:telemetry-notify-event" + + def __init__(self, conn, config, store, update_interval): + self._conn = conn + self._config = config + self._store = store + self._update_interval = update_interval + self._path_parser = PathParser(self._conn.ctx) + self._id = self._config["id"] + self._updates_only = False + self._subscriptions = {} + self._parse_config() + self._validate_config() + + def _parse_config(self): + request_config = self._config.get("config") + if request_config is None: + request_config = {} + self._updates_only = request_config.get("updates-only") + if self._updates_only is None: + self._updates_only = False + subscriptions = self._config.get("subscriptions") + if subscriptions is None: + subscriptions = {} + subscriptions = subscriptions.get("subscription") + if subscriptions is None: + subscriptions = [] + for subscription in subscriptions: + sid = subscription.get("id") + if sid is None: + continue + subscription_config = subscription.get("config") + parsed_subscription = { + "id": subscription_config.get("id"), + "path": subscription_config.get("path"), + "mode": subscription_config.get("mode"), + "sample-interval": subscription_config.get("sample-interval"), + "suppress-redundant": subscription_config.get("suppress-redundant"), + "heartbeat-interval": subscription_config.get("heartbeat-interval"), + } + self._subscriptions[sid] = parsed_subscription + + def _validate_config(self): + for _, config in self._subscriptions.items(): + if config["path"] is None: + msg = "path is mandatory" + logger.error("Subscription config validation failed: %s", msg) + raise ValidationFailedError(msg) + if not self._path_parser.is_valid_path(config["path"]): + msg = f"invalid path: {config['path']}" + logger.error("Subscription config validation failed: %s", msg) + raise ValidationFailedError(msg) + + def _get_data(self, xpath): + data = self._conn.get_operational(xpath, strip=False) + # NOTE: Connector returns a value None instead of raising an exception if the data was not found. + if data is None: + logger.info("data for path %s is not found.", xpath) + data = {} + return self._path_parser.parse_dict_into_leaves(data, xpath) + + def _send_notification(self, notif): + """Send a notification. + + Args: + notif (dict): Notification to send. + """ + self._conn.send_notification(self.NOTIF_PATH, notif) + + def _send_sync_response(self): + notif = { + "type": "SYNC_RESPONSE", + "request-id": self._id, + } + self._send_notification(notif) + + def _retrieve_current_data(self): + for sid, subscription in self._subscriptions.items(): + path = subscription["path"] + data = self._get_data(path) + for sub_path, value in data.items(): + self._store.set((self._id, sid), sub_path, value) + + def _send_current_data(self): + for sid, _ in self._subscriptions.items(): + ids = (self._id, sid) + sub_paths = self._store.list(ids) + for sub_path in sub_paths: + data = self._store.get(ids, sub_path) + notif = { + "type": "UPDATE", + "request-id": self._id, + "subscription-id": sid, + "path": sub_path, + "json-data": json.dumps(data["value"]), + } + self._send_notification(notif) + + async def start(self): + """Start the subscription.""" + # Start session in __init__() because it will be used to parse and validate configuration parameters. + self._retrieve_current_data() + if not self._updates_only: + self._send_current_data() + self._send_sync_response() + + async def stop(self): + """Stop the subscription.""" + pass + + def get_state(self): + """Get subscription state. + + Returns: + dict: subscription state data. + """ + subscriptions = [] + for sid, subscription in self._subscriptions.items(): + subscriptions.append( + { + "id": sid, + "path": subscription["path"], + "mode": subscription["mode"], + "sample-interval": subscription["sample-interval"], + "suppress-redundant": subscription["suppress-redundant"], + "heartbeat-interval": subscription["heartbeat-interval"], + } + ) + return { + "id": self._config["id"], + "mode": self._config["config"]["mode"], + "updates-only": self._updates_only, + "subscriptions": subscriptions, + } + + +class StreamSubscription(Subscription): + """Subscription for the STREAM mode.""" + + HEARTBEAT_DISABLED = 0 + + def __init__(self, conn, config, store, update_interval): + self._default_sampling_interval = update_interval * 2 + super().__init__(conn, config, store, update_interval) + self._loop_tasks = {} + + def _target_defined_mode(self, path): + # NOTE: Select the mode by provided path. + return "SAMPLE" + + def _parse_config(self): + super()._parse_config() + for _, subscription in self._subscriptions.items(): + if subscription["sample-interval"] is None: + subscription["sample-interval"] = self._default_sampling_interval + if subscription["suppress-redundant"] is None: + subscription["suppress-redundant"] = False + if subscription["heartbeat-interval"] is None: + subscription["heartbeat-interval"] = self.HEARTBEAT_DISABLED + if subscription["mode"] == "TARGET_DEFINED": + subscription["mode"] = self._target_defined_mode(subscription["path"]) + + def _validate_config(self): + super()._validate_config() + for _, config in self._subscriptions.items(): + if config["mode"] is None: + msg = "mode is mandatory" + logger.error("Subscription config validation failed: %s", msg) + raise ValidationFailedError(msg) + if ( + config["heartbeat-interval"] < self._update_interval + and config["heartbeat-interval"] != self.HEARTBEAT_DISABLED + ): + msg = f"heartbeat-interval is shorter than minimum interval {self._update_interval}" + logger.error("Subscription config validation failed: %s", msg) + raise ValidationFailedError(msg) + if config["mode"] == "SAMPLE": + if config["sample-interval"] < self._update_interval: + msg = f"sample-interval is shorter than minimum interval {self._update_interval}" + logger.error("Subscription config validation failed: %s", msg) + raise ValidationFailedError(msg) + + async def start(self): + await super().start() + loops = { + "ON_CHANGE": self._on_change_loop, + "SAMPLE": self._sample_loop, + } + for _, subscription in self._subscriptions.items(): + try: + self._loop_tasks[subscription["id"]] = asyncio.create_task( + loops[subscription["mode"]](subscription) + ) + except KeyError: + continue + + async def stop(self): + for _, loop_task in self._loop_tasks.items(): + loop_task.cancel() + for _, loop_task in self._loop_tasks.items(): + while True: + if loop_task.done(): + break + await asyncio.sleep(0.1) + await super().stop() + + def _should_send_notif(self, config, ids, sub_path, value): + send_notif = True + suppress_redundant = ( + config["suppress-redundant"] or config["mode"] == "ON_CHANGE" + ) + hb = timedelta(microseconds=config["heartbeat-interval"] / 1000) + if suppress_redundant: + try: + prev_data = self._store.get(ids, sub_path) + hb_expired = False + if hb > timedelta(0): + hb_expired = (datetime.now() - prev_data["update-time"]) > hb + if value == prev_data["value"] and not hb_expired: + send_notif = False + except TelemetryNotExistError: + # The data node of the sub_path is created. + pass + return send_notif + + def _sample_and_notify(self, config): + ids = (self._id, config["id"]) + data = self._get_data(config["path"]) + currents = set(self._store.list(ids)) + exists = set() + # Created or updated data nodes. + for sub_path, value in data.items(): + exists.add(sub_path) + if self._should_send_notif(config, ids, sub_path, value): + self._store.set(ids, sub_path, value) + notif = { + "type": "UPDATE", + "request-id": self._id, + "subscription-id": config["id"], + "path": sub_path, + "json-data": json.dumps(value), + } + self._send_notification(notif) + # Deleted data nodes. + for sub_path in currents - exists: + try: + self._store.delete(ids, sub_path) + except TelemetryNotExistError: + pass + notif = { + "type": "DELETE", + "request-id": self._id, + "subscription-id": config["id"], + "path": sub_path, + } + self._send_notification(notif) + + async def _on_change_loop(self, config): + while True: + # NOTE: We should subscribe state change notifications with the central datastore if it is possible. To do + # it, we need to design the archtecture and implement it into the model server daemons. Until then, we + # will use this polling implementation. + await asyncio.sleep(self._update_interval / 1000 / 1000 / 1000) + try: + self._sample_and_notify(config) + except Exception as e: + logger.error( + "Failed to update current state and send notification. %s: %s", + type(e).__name__, + e, + ) + + async def _sample_loop(self, config): + while True: + await asyncio.sleep(config["sample-interval"] / 1000 / 1000 / 1000) + try: + self._sample_and_notify(config) + except Exception as e: + logger.error( + "Failed to update current state and send notification. %s: %s", + type(e).__name__, + e, + ) + + +class OnceSubscription(Subscription): + """Subscription for the ONCE mode.""" + + pass + + +class PollSubscription(Subscription): + """Subscription for the POLL mode.""" + + async def poll_cb(self, xpath, inputs, event, priv): + """Callback function for a poll request. + + Args: + xpath (str): Full data path of the request. + inputs (dict): Input parameters. + event (str): Event type of the callback. It is always "rpc". Don't care. + priv (any): Private data from the request subscribing. + """ + self._retrieve_current_data() + self._send_current_data() + self._send_sync_response() + + +class SubscribeRequestTypedHandler: + """Base handler class for each change types of subscribe-request. + + Args: + rid (int): Identification of the subscribe-request. + change (sysrepo.Change): Change to apply. + """ + + def __init__(self, rid, change): + self._id = rid + self._change = change + + def validate(self, user): + """Validate the change. + + Args: + user (dict): User defined data. + """ + pass + + async def apply(self, user): + """Apply the change. + + Args: + user (dict): User defined data. + """ + pass + + async def revert(self, user): + """Revert the change. + + Args: + user (dict): User defined data. + """ + pass + + +class SubscribeRequestCreatedHandler(SubscribeRequestTypedHandler): + """Handler for a created subscribe-request.""" + + SUBSCRIPTIONS = { + "STREAM": StreamSubscription, + "ONCE": OnceSubscription, + "POLL": PollSubscription, + } + + def __init__(self, rid, change): + super().__init__(rid, change) + self._config = self._change.value + + def validate(self, user): + try: + mode = self._config["config"]["mode"] + except KeyError as e: + msg = "mode should be specified" + logger.error(msg) + raise sysrepo.SysrepoInvalArgError(msg) from e + try: + self._subscription = self.SUBSCRIPTIONS[mode]( + user["conn"], + self._config, + user["telemetry-store"], + user["update-interval"], + ) + except KeyError as e: + msg = f"invalid mode {mode}" + logger.error(msg) + raise sysrepo.SysrepoInvalArgError(msg) from e + except ValidationFailedError as e: + msg = f"invalid subscription parameter: {e.msg}" + logger.error(msg) + raise sysrepo.SysrepoInvalArgError(msg) from e + + async def apply(self, user): + user["subscription-store"].add(self._id, self._subscription) + await self._subscription.start() + + async def revert(self, user): + await self._subscription.stop() + user["subscription-store"].delete(self._id) + + +class SubscribeRequestModifiedHandler(SubscribeRequestTypedHandler): + """Handler for a modified subscribe-request.""" + + def __init__(self, rid, change): + super().__init__(rid, change) + msg = "subscription modification is not supported" + logger.error(msg) + raise sysrepo.SysrepoUnsupportedError(msg) + + +class SubscribeRequestDeletedHandler(SubscribeRequestTypedHandler): + """Handler for a deleted subscribe-request.""" + + def validate(self, user): + try: + self._subscription = user["subscription-store"].get(self._id) + except SubscriptionNotExistError as e: + msg = f"invalid id {self._id}" + logger.error(msg) + raise sysrepo.SysrepoInvalArgError(msg) from e + + async def apply(self, user): + await self._subscription.stop() + user["subscription-store"].delete(self._id) + + async def revert(self, user): + user["subscription-store"].add(self._id, self._subscription) + await self._subscription.start() + + +class SubscribeRequestChangeHandler(ChangeHandler): + """ChangeHndler for a subscribe-request.""" + + TYPES = { + "created": SubscribeRequestCreatedHandler, + "modified": SubscribeRequestModifiedHandler, + "deleted": SubscribeRequestDeletedHandler, + } + + def __init__(self, server, change): + super().__init__(server, change) + self.xpath = list(libyang.xpath_split(change.xpath)) + self._noop = False + if not ( + len(self.xpath) == 2 + and self.xpath[0][0] == "goldstone-telemetry" + and self.xpath[0][1] == "subscribe-requests" + and self.xpath[1][1] == "subscribe-request" + and self.xpath[1][2][0][0] == "id" + ): + self._noop = True + if not self._noop: + logger.debug("SubscribeRequestChangeHandler: %s", change) + self._handler = self.TYPES[self.type](int(self.xpath[1][2][0][1]), self.change) + + def validate(self, user): + if self._noop: + return + self._handler.validate(user) + + async def apply(self, user): + if self._noop: + return + await self._handler.apply(user) + + async def revert(self, user): + if self._noop: + return + await self._handler.revert(user) + + +class TelemetryServer(ServerBase): + """goldstone-terlemetry server. + + The server manages telemetry subscriptions requested via goldstone-telemetry. + + Args: + subscription_store (store.SubscriptionStore): Datastore for managed subscriptions. + telemetry_store (store.TelemetryStore): Datastore for telemetry data. + update_interval (int): Telemetry data update interval in seconds. + """ + + DEFAULT_UPDATE_INTERVAL = 5 + + def __init__( + self, + conn, + subscription_store, + telemetry_store, + update_interval=DEFAULT_UPDATE_INTERVAL, + ): + super().__init__(conn, "goldstone-telemetry") + self._subscription_store = subscription_store + self._telemetry_store = telemetry_store + self._update_interval = update_interval * 1000 * 1000 * 1000 + self.handlers = { + "subscribe-requests": {"subscribe-request": SubscribeRequestChangeHandler} + } + + async def start(self): + """Start a service.""" + tasks = await super().start() + # NOTE: The sysrepo v1 doesn't support subscriptions to specific "data" instances. It supports subscriptions to + # "schema" nodes. So, we should share a subscription to the RPC "/poll" for all POLL mode subscribe requests. + # The sysrepo v2 supports subscriptions to specific "data" instances. Then, we can subscribe an action for a + # data instance of a subscribe request like "/subscribe-requests/subscribe-request[id='{request-id}']/poll". + # See also: + # - https://github.com/sysrepo/sysrepo/issues/1255 + # - https://github.com/sysrepo/sysrepo/issues/1438 + xpath = "/goldstone-telemetry:poll" + self.conn.subscribe_rpc_call(xpath, self.poll_cb) + return tasks + + async def stop(self): + """Stop a service.""" + for rid in self._subscription_store.list(): + await self._subscription_store.get(rid).stop() + super().stop() + + def pre(self, user): + """Pre action for changes.""" + user["conn"] = self.conn.conn + user["subscription-store"] = self._subscription_store + user["telemetry-store"] = self._telemetry_store + user["update-interval"] = self._update_interval + + async def poll_cb(self, xpath, inputs, event, priv): + """Callback function for a poll request. + + Args: + xpath (str): Full data path of the request. + inputs (dict): Input parameters. + event (str): Event type of the callback. It is always "rpc". Don't care. + priv (any): Private data from the request subscribing. + """ + logger.info( + "poll_cb - xpath: %s, inputs: %s, event: %s", + xpath, + inputs, + event, + ) + rid = inputs["id"] + logger.info("Poll request for %s.", rid) + subscription = self._subscription_store.get(rid) + await subscription.poll_cb(xpath, inputs, event, priv) + + async def oper_cb(self, xpath, priv): + """Callback function for a operational state request. + + Args: + xpath (str): Full data path of the request. + priv (any): Private data from the request subscribing. + """ + subscribe_requests = [] + for rid in self._subscription_store.list(): + subscription = self._subscription_store.get(rid) + data = subscription.get_state() + subscribe_request = { + "id": data["id"], + "state": { + "id": data["id"], + "mode": data["mode"], + }, + } + if data["updates-only"] is not None: + subscribe_request["state"]["updates-only"] = data["updates-only"] + internal_subscriptions = [] + for internal_subscription_data in data["subscriptions"]: + internal_subscription = { + "id": internal_subscription_data["id"], + "state": { + "id": internal_subscription_data["id"], + "path": internal_subscription_data["path"], + }, + } + if internal_subscription_data["mode"] is not None: + internal_subscription["state"]["mode"] = internal_subscription_data[ + "mode" + ] + if internal_subscription_data["sample-interval"] is not None: + internal_subscription["state"][ + "sample-interval" + ] = internal_subscription_data["sample-interval"] + if internal_subscription_data["suppress-redundant"] is not None: + internal_subscription["state"][ + "suppress-redundant" + ] = internal_subscription_data["suppress-redundant"] + if internal_subscription_data["heartbeat-interval"] is not None: + internal_subscription["state"][ + "heartbeat-interval" + ] = internal_subscription_data["heartbeat-interval"] + internal_subscriptions.append(internal_subscription) + if len(internal_subscriptions) > 0: + subscribe_request["subscriptions"] = { + "subscription": internal_subscriptions + } + subscribe_requests.append(subscribe_request) + return { + "subscribe-requests": { + "subscribe-request": subscribe_requests, + } + } diff --git a/src/system/telemetry/requirements.txt b/src/system/telemetry/requirements.txt new file mode 100644 index 00000000..ee4ba4f3 --- /dev/null +++ b/src/system/telemetry/requirements.txt @@ -0,0 +1 @@ +aiohttp diff --git a/src/system/telemetry/setup.py b/src/system/telemetry/setup.py new file mode 100644 index 00000000..d2f17050 --- /dev/null +++ b/src/system/telemetry/setup.py @@ -0,0 +1,20 @@ +import setuptools + +with open("requirements.txt", "r") as f: + install_requires = f.read().split() + +setuptools.setup( + name="goldstone_system_telemetry", + version="0.1.0", + install_requires=install_requires, + description="Streaming telemetry server", + url="https://github.com/oopt-goldstone/goldstone-mgmt", + python_requires=">=3.7", + entry_points={ + "console_scripts": [ + "gssystemd-telemetry = goldstone.system.telemetry.main:main", + ], + }, + packages=["goldstone.system.telemetry"], + zip_safe=False, +) diff --git a/src/system/telemetry/tests/__init__.py b/src/system/telemetry/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/system/telemetry/tests/test.py b/src/system/telemetry/tests/test.py new file mode 100644 index 00000000..b2067a86 --- /dev/null +++ b/src/system/telemetry/tests/test.py @@ -0,0 +1,12 @@ +"""Unittest runner for a streaming telemetry server.""" + + +import unittest +import sys + + +if __name__ == "__main__": + sys.path.insert(0, "../../../lib/") + sys.path.insert(0, "../") + testsuite = unittest.TestLoader().discover(".") + unittest.TextTestRunner(verbosity=2).run(testsuite) diff --git a/src/system/telemetry/tests/test_path.py b/src/system/telemetry/tests/test_path.py new file mode 100644 index 00000000..70db01ff --- /dev/null +++ b/src/system/telemetry/tests/test_path.py @@ -0,0 +1,57 @@ +"""Tests for path utilities.""" + + +import unittest +from goldstone.lib.connector.sysrepo import Connector +from goldstone.system.telemetry.path import PathParser + + +class TestPathParser(unittest.TestCase): + """Tests for PathParser.""" + + def setUp(self): + self.conn = Connector() + self.ctx = self.conn.ctx + + def tearDown(self) -> None: + self.conn.stop() + + def test_valid_path(self): + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/name" + p = PathParser(self.ctx) + self.assertTrue(p.is_valid_path(path)) + + def test_invalid_path(self): + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/unknown-node" + p = PathParser(self.ctx) + self.assertFalse(p.is_valid_path(path)) + + def test_parse_dict(self): + data = { + "interfaces": { + "interface": [ + { + "name": "Interface1/0/1", + "config": { + "name": "Interface1/0/1", + "admin-status": "UP", + }, + "state": { + "name": "Interface1/0/1", + "admin-status": "DOWN", + }, + } + ] + } + } + path = ( + "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config" + ) + p = PathParser(self.ctx) + parsed_data = p.parse_dict_into_leaves(data, path) + expected = {path + "/name": "Interface1/0/1", path + "/admin-status": "UP"} + self.assertEqual(parsed_data, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/system/telemetry/tests/test_store.py b/src/system/telemetry/tests/test_store.py new file mode 100644 index 00000000..1caf4aaa --- /dev/null +++ b/src/system/telemetry/tests/test_store.py @@ -0,0 +1,162 @@ +"""Tests for datastores.""" + +import unittest +import datetime +from goldstone.lib.connector.sysrepo import Connector +from goldstone.system.telemetry.store import ( + InMemoryTelemetryStore, + TelemetryNotExistError, + InMemorySubscriptionStore, + SubscriptionExistError, + SubscriptionNotExistError, +) +from goldstone.system.telemetry.telemetry import Subscription + + +class TestInMemoryTelemetryStore(unittest.TestCase): + """Tests for InMemoryTelemetryStore.""" + + def test_set(self): + ts = InMemoryTelemetryStore() + ids = (1, 1) + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + value = "UP" + # Create an entry. + self.assertFalse(path in ts.list(ids)) + before = datetime.datetime.now() + ts.set(ids, path, value) + after = datetime.datetime.now() + self.assertTrue(path in ts.list(ids)) + stored_telemetry = ts.get(ids, path) + self.assertEqual(stored_telemetry["value"], value) + self.assertTrue(before <= stored_telemetry["update-time"] <= after) + # Update an entry. + new_value = "DOWN" + self.assertTrue(path in ts.list(ids)) + before = datetime.datetime.now() + ts.set(ids, path, new_value) + after = datetime.datetime.now() + self.assertTrue(path in ts.list(ids)) + stored_telemetry = ts.get(ids, path) + self.assertEqual(stored_telemetry["value"], new_value) + self.assertTrue(before <= stored_telemetry["update-time"] <= after) + + def test_delete(self): + ts = InMemoryTelemetryStore() + ids = (1, 1) + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + value = "UP" + ts.set(ids, path, value) + self.assertTrue(path in ts.list(ids)) + ts.delete(ids, path) + self.assertFalse(path in ts.list(ids)) + + def test_delete_not_exist(self): + ts = InMemoryTelemetryStore() + ids = (1, 1) + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + self.assertFalse(path in ts.list(ids)) + with self.assertRaises(TelemetryNotExistError): + ts.delete(ids, path) + + def test_get(self): + ts = InMemoryTelemetryStore() + ids = (1, 1) + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + value = "UP" + ts.set(ids, path, value) + self.assertTrue(path in ts.list(ids)) + stored_telemetry = ts.get(ids, path) + self.assertEqual(stored_telemetry["value"], value) + + def test_get_not_exist(self): + ts = InMemoryTelemetryStore() + ids = (1, 1) + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + self.assertFalse(path in ts.list(ids)) + with self.assertRaises(TelemetryNotExistError): + ts.get(ids, path) + + +class TestInMemorySubscriptionStore(unittest.TestCase): + """Tests for InMemorySubscriptionStore.""" + + def test_add(self): + conn = Connector() + ss = InMemorySubscriptionStore() + ts = InMemoryTelemetryStore() + id_1 = 1 + subscription = Subscription(conn, {"id": id_1}, ts, 5) + ss.add(id_1, subscription) + self.assertTrue(id_1 in ss.list()) + stored_subscription = ss.get(id_1) + self.assertEqual(stored_subscription, subscription) + + def test_add_exist(self): + conn = Connector() + ss = InMemorySubscriptionStore() + ts = InMemoryTelemetryStore() + id_1 = 1 + subscription = Subscription(conn, {"id": id_1}, ts, 5) + ss.add(id_1, subscription) + with self.assertRaises(SubscriptionExistError): + ss.add(id_1, subscription) + + def test_delete(self): + conn = Connector() + ss = InMemorySubscriptionStore() + ts = InMemoryTelemetryStore() + id_1 = 1 + subscription = Subscription(conn, {"id": id_1}, ts, 5) + ss.add(id_1, subscription) + self.assertTrue(id_1 in ss.list()) + ss.delete(id_1) + self.assertFalse(id_1 in ss.list()) + + def test_delete_not_exist(self): + ss = InMemorySubscriptionStore() + with self.assertRaises(SubscriptionNotExistError): + ss.delete(1) + + def test_get(self): + conn = Connector() + ss = InMemorySubscriptionStore() + ts = InMemoryTelemetryStore() + id_1 = 1 + subscription_1 = Subscription(conn, {"id": id_1}, ts, 5) + ss.add(id_1, subscription_1) + id_2 = 2 + subscription_2 = Subscription(conn, {"id": id_2}, ts, 5) + ss.add(id_2, subscription_2) + stored_subscription_1 = ss.get(id_1) + stored_subscription_2 = ss.get(id_2) + self.assertEqual(stored_subscription_1, subscription_1) + self.assertEqual(stored_subscription_2, subscription_2) + + def test_get_not_exist(self): + conn = Connector() + ss = InMemorySubscriptionStore() + ts = InMemoryTelemetryStore() + id_1 = 1 + subscription_1 = Subscription(conn, {"id": id_1}, ts, 5) + ss.add(id_1, subscription_1) + with self.assertRaises(SubscriptionNotExistError): + ss.delete(2) + + def test_list(self): + conn = Connector() + ss = InMemorySubscriptionStore() + ts = InMemoryTelemetryStore() + self.assertEqual(ss.list(), []) + id_1 = 1 + subscription = Subscription(conn, {"id": id_1}, ts, 5) + ss.add(id_1, subscription) + self.assertEqual(ss.list(), [id_1]) + id_2 = 2 + subscription = Subscription(conn, {"id": id_2}, ts, 5) + ss.add(id_2, subscription) + self.assertEqual(ss.list(), [id_1, id_2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/system/telemetry/tests/test_telemetry.py b/src/system/telemetry/tests/test_telemetry.py new file mode 100644 index 00000000..9fcf70d6 --- /dev/null +++ b/src/system/telemetry/tests/test_telemetry.py @@ -0,0 +1,1809 @@ +"""Tests for subscription stores.""" + + +import unittest +import asyncio +import logging +import time +import sysrepo +from multiprocessing import Process, Queue +from goldstone.lib.core import ServerBase, NoOp +from goldstone.lib.connector.sysrepo import Connector +from goldstone.system.telemetry.store import ( + InMemorySubscriptionStore, + InMemoryTelemetryStore, +) +from goldstone.system.telemetry.telemetry import TelemetryServer + + +class MockGSServer(ServerBase): + """MockGSServer is mock handler server for Goldstone primitive models. + + Attributes: + oper_data (dict): Data for oper_cb() to return. You can set this to configure mock's behavior. + """ + + def __init__(self, conn, module): + super().__init__(conn, module) + self.oper_data = {} + self.handlers = {"interfaces": NoOp} + + def oper_cb(self, xpath, priv): + return self.oper_data + + +class MockGSInterfaceServer(MockGSServer): + def __init__(self, conn): + super().__init__(conn, "goldstone-interfaces") + + +MOCK_SERVERS = { + "goldstone-interfaces": MockGSInterfaceServer, +} + + +def run_mock_server(q, mock_modules): + """Run mock servers. + + A TestCase can communicate with MockServers by using a Queue. + Stop MockServers: {"type": "stop"} + Set operational state data of a MockServer: {"type": "set", "server": "", "data": ""} + + Args: + q (Queue): Queue to communicate between a TestCase and MockServers. + mock_modules (list of str): Names of modules to mock. Keys in MOCK_SERVERS. + """ + conn = Connector() + servers = {} + for mock_module in mock_modules: + servers[mock_module] = MOCK_SERVERS[mock_module](conn) + + async def _main(): + tasks = [] + for server in servers.items(): + tasks += await server[1].start() + + async def evloop(): + while True: + await asyncio.sleep(0.01) + try: + msg = q.get(False) + except: + pass + else: + if msg["type"] == "stop": + return + elif msg["type"] == "set-oper-data": + servers[msg["server"]].oper_data = msg["data"] + + tasks.append(evloop()) + tasks = [ + t if isinstance(t, asyncio.Task) else asyncio.create_task(t) for t in tasks + ] + + done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for task in done: + e = task.exception() + if e: + raise e + + asyncio.run(_main()) + + +def config_subscription(sess, params): + sess.switch_datastore("running") + rid = params["id"] + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/config/id", + rid, + ) + if params["mode"] is not None: + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/config/mode", + params["mode"], + ) + if params["updates-only"] is not None: + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/config/updates-only", + params["updates-only"], + ) + for subscription in params["subscriptions"]: + sid = subscription["id"] + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/id", + sid, + ) + if subscription["path"] is not None: + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/path", + subscription["path"], + ) + if subscription["mode"] is not None: + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/mode", + subscription["mode"], + ) + if subscription["sample-interval"] is not None: + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/sample-interval", + subscription["sample-interval"], + ) + if subscription["suppress-redundant"] is not None: + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/suppress-redundant", + subscription["suppress-redundant"], + ) + if subscription["heartbeat-interval"] is not None: + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/heartbeat-interval", + subscription["heartbeat-interval"], + ) + sess.apply_changes() + + +class TestTelemetryServer(unittest.IsolatedAsyncioTestCase): + """Tests for TelemetryServer.""" + + MOCK_WAIT = 1 + NOTIFICATION_WAIT = 0.1 + + async def asyncSetUp(self): + logging.basicConfig(level=logging.CRITICAL) + # NOTE: Enable for debugging. + # logging.basicConfig(level=logging.DEBUG) + # self.maxDiff = None + self.conn = Connector() + + self.conn.delete_all("goldstone-telemetry") + self.conn.apply() + + self.received_notif = {} + self.ss = InMemorySubscriptionStore() + self.ts = InMemoryTelemetryStore() + self.server = TelemetryServer(self.conn, self.ss, self.ts) + self.q = Queue() + mock_modules = ["goldstone-interfaces"] + self.process = Process(target=run_mock_server, args=(self.q, mock_modules)) + self.process.start() + + self.tasks = await self.server.start() + + async def asyncTearDown(self): + await self.server.stop() + self.tasks = [] + self.conn.stop() + self.q.put({"type": "stop"}) + self.process.join() + self.clear_received_notif() + + async def run_test(self, test): + """Run a test as a thread. + + Args: + test (func): Test to run. + """ + self.tasks.append(asyncio.to_thread(test)) + tasks = [ + t if isinstance(t, asyncio.Task) else asyncio.create_task(t) + for t in self.tasks + ] + + done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for task in done: + e = task.exception() + if e: + raise e + + def set_mock_oper_data(self, server, data): + """Set operational state data to the mock server. + + Args: + server (str): Target mock server name. A key in MOCK_SERVERS. + data (dict): Operational state data that the server returns. + """ + self.q.put({"type": "set-oper-data", "server": server, "data": data}) + + def notif_callback(self, xpath, notif_type, notif, ts, priv): + if "path" in notif.keys(): + self.received_notif[notif["path"]] = notif + else: + self.received_notif["sync-response"] = notif + + def clear_received_notif(self): + self.received_notif = {} + + async def test_empty(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + expected = {} + self.assertEqual(data, expected) + + await self.run_test(test) + + async def test_basic_op_stream_sample_subscription(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "STREAM", + "updates-only": False, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "suppress-redundant": True, + "heartbeat-interval": 10 * 1000 * 1000 * 1000, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + expected = { + "subscribe-requests": { + "subscribe-request": [ + { + "id": params["id"], + "config": { + "id": params["id"], + "mode": params["mode"], + "updates-only": params["updates-only"], + }, + "state": { + "id": params["id"], + "mode": params["mode"], + "updates-only": params["updates-only"], + }, + "subscriptions": { + "subscription": [ + { + "id": s["id"], + "config": { + "id": s["id"], + "path": s["path"], + "mode": s["mode"], + "sample-interval": s[ + "sample-interval" + ], + "suppress-redundant": s[ + "suppress-redundant" + ], + "heartbeat-interval": s[ + "heartbeat-interval" + ], + }, + "state": { + "id": s["id"], + "path": s["path"], + "mode": s["mode"], + "sample-interval": s[ + "sample-interval" + ], + "suppress-redundant": s[ + "suppress-redundant" + ], + "heartbeat-interval": s[ + "heartbeat-interval" + ], + }, + } + ] + }, + } + ] + } + } + self.assertEqual(data, expected) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"UP"', + }, + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Update the target data. + sess.switch_datastore("running") + sess.set_item(path, "DOWN") + sess.apply_changes() + + # Wait sample interval. + time.sleep(s["sample-interval"] / 1000 / 1000 / 1000) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"DOWN"', + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + # Modify a subscription. + sess.switch_datastore("running") + new_path = path_prefix + "/config/interface-type" + rid = params["id"] + sid = s["id"] + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/path", + new_path, + ) + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + sess.apply_changes() + sess.discard_changes() + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + self.assertEqual(data, expected) + + # Delete a subscription. + sess.switch_datastore("running") + sess.delete_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']" + ) + sess.apply_changes() + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + expected_after_delete = {} + self.assertEqual(data, expected_after_delete) + + await self.run_test(test) + + async def test_basic_op_stream_on_change_subscription(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "STREAM", + "updates-only": False, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "ON_CHANGE", + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": 20 * 1000 * 1000 * 1000, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + expected = { + "subscribe-requests": { + "subscribe-request": [ + { + "id": params["id"], + "config": { + "id": params["id"], + "mode": params["mode"], + "updates-only": params["updates-only"], + }, + "state": { + "id": params["id"], + "mode": params["mode"], + "updates-only": params["updates-only"], + }, + "subscriptions": { + "subscription": [ + { + "id": s["id"], + "config": { + "id": s["id"], + "path": s["path"], + "mode": s["mode"], + "heartbeat-interval": s[ + "heartbeat-interval" + ], + }, + "state": { + "id": s["id"], + "path": s["path"], + "mode": s["mode"], + "sample-interval": 10 + * 1000 + * 1000 + * 1000, + "suppress-redundant": False, + "heartbeat-interval": s[ + "heartbeat-interval" + ], + }, + } + ] + }, + } + ] + } + } + self.assertEqual(data, expected) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"UP"', + }, + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Update the target data. + sess.switch_datastore("running") + sess.set_item(path, "DOWN") + sess.apply_changes() + + # Wait default update interval. + time.sleep(5) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"DOWN"', + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + # Modify a subscription. + sess.switch_datastore("running") + new_path = path_prefix + "/config/interface-type" + rid = params["id"] + sid = s["id"] + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/path", + new_path, + ) + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + sess.apply_changes() + sess.discard_changes() + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + self.assertEqual(data, expected) + + # Delete a subscription. + sess.switch_datastore("running") + sess.delete_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']" + ) + sess.apply_changes() + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + expected_after_delete = {} + self.assertEqual(data, expected_after_delete) + + await self.run_test(test) + + async def test_basic_op_poll_subscription(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "POLL", + "updates-only": False, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": None, + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + expected = { + "subscribe-requests": { + "subscribe-request": [ + { + "id": params["id"], + "config": { + "id": params["id"], + "mode": params["mode"], + "updates-only": params["updates-only"], + }, + "state": { + "id": params["id"], + "mode": params["mode"], + "updates-only": params["updates-only"], + }, + "subscriptions": { + "subscription": [ + { + "id": s["id"], + "config": { + "id": s["id"], + "path": s["path"], + }, + "state": { + "id": s["id"], + "path": s["path"], + }, + } + ] + }, + } + ] + } + } + self.assertEqual(data, expected) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"UP"', + }, + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Update the target data. + sess.switch_datastore("running") + sess.set_item(path, "DOWN") + sess.apply_changes() + + # Send a poll request. + sess.rpc_send("/goldstone-telemetry:poll", {"id": params["id"]}) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"DOWN"', + }, + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + # Modify a subscription. + sess.switch_datastore("running") + new_path = path_prefix + "/config/interface-type" + rid = params["id"] + sid = s["id"] + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/path", + new_path, + ) + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + sess.apply_changes() + sess.discard_changes() + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + self.assertEqual(data, expected) + + # Delete a subscription. + sess.switch_datastore("running") + sess.delete_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']" + ) + sess.apply_changes() + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + expected_after_delete = {} + self.assertEqual(data, expected_after_delete) + + await self.run_test(test) + + async def test_basic_op_once_subscription(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "ONCE", + "updates-only": False, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": None, + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + expected = { + "subscribe-requests": { + "subscribe-request": [ + { + "id": params["id"], + "config": { + "id": params["id"], + "mode": params["mode"], + "updates-only": params["updates-only"], + }, + "state": { + "id": params["id"], + "mode": params["mode"], + "updates-only": params["updates-only"], + }, + "subscriptions": { + "subscription": [ + { + "id": s["id"], + "config": { + "id": s["id"], + "path": s["path"], + }, + "state": { + "id": s["id"], + "path": s["path"], + }, + } + ] + }, + } + ] + } + } + self.assertEqual(data, expected) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"UP"', + }, + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + # Modify a subscription. + sess.switch_datastore("running") + new_path = path_prefix + "/config/interface-type" + rid = params["id"] + sid = s["id"] + sess.set_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']/subscriptions" + f"/subscription[id='{sid}']/config/path", + new_path, + ) + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + sess.apply_changes() + sess.discard_changes() + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + self.assertEqual(data, expected) + + # Delete a subscription. + sess.switch_datastore("running") + sess.delete_item( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{rid}']" + ) + sess.apply_changes() + sess.switch_datastore("operational") + data = sess.get_data("/goldstone-telemetry:subscribe-requests") + expected_after_delete = {} + self.assertEqual(data, expected_after_delete) + + await self.run_test(test) + + async def test_config_subscription_ok(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + params = { + "id": 1, + "mode": "STREAM", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "suppress-redundant": True, + "heartbeat-interval": 10 * 1000 * 1000 * 1000, + } + ], + } + config_subscription(sess, params) + + await self.run_test(test) + + async def test_config_subscription_error_no_mode(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + params = { + "id": 1, + "mode": None, + "updates-only": None, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + config_subscription(sess, params) + + await self.run_test(test) + + async def test_config_subscription_error_no_subscription_path(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + params = { + "id": 1, + "mode": "STREAM", + "updates-only": None, + "subscriptions": [ + { + "id": 1, + "path": None, + "mode": "SAMPLE", + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + config_subscription(sess, params) + + await self.run_test(test) + + async def test_config_subscription_error_invalid_path(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + path = ( + "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config" + "/unknown-leaf-node-to-fail" + ) + params = { + "id": 1, + "mode": "STREAM", + "updates-only": None, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + config_subscription(sess, params) + + await self.run_test(test) + + async def test_config_subscription_error_no_stream_mode(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + params = { + "id": 1, + "mode": "STREAM", + "updates-only": None, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": None, + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + config_subscription(sess, params) + + await self.run_test(test) + + async def test_config_subscription_error_short_sample_interval(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + params = { + "id": 1, + "mode": "STREAM", + "updates-only": None, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": 4 * 1000 * 1000 * 1000, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + config_subscription(sess, params) + + await self.run_test(test) + + async def test_config_subscription_error_short_heartbeat_interval(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config/admin-status" + params = { + "id": 1, + "mode": "STREAM", + "updates-only": None, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": 4 * 1000 * 1000 * 1000, + } + ], + } + with self.assertRaises(sysrepo.SysrepoCallbackFailedError): + config_subscription(sess, params) + + await self.run_test(test) + + async def test_subscribe_container(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']/config" + path_name = path + "/name" + path_admin_status = path + "/admin-status" + sess.switch_datastore("running") + sess.set_item(path_name, "Interface1/0/1") + sess.set_item(path_admin_status, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "ONCE", + "updates-only": False, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": None, + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + path_name: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": path_name, + "json-data": '"Interface1/0/1"', + }, + path_admin_status: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": path_admin_status, + "json-data": '"UP"', + }, + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + async def test_subscribe_container_list(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path = "/goldstone-interfaces:interfaces/interface" + path_1 = path + "[name='Interface1/0/1']" + path_name_1 = path_1 + "/name" + path_config_name_1 = path_1 + "/config/name" + path_config_admin_status_1 = path_1 + "/config/admin-status" + path_2 = path + "[name='Interface1/0/2']" + path_name_2 = path_2 + "/name" + path_config_name_2 = path_2 + "/config/name" + path_config_admin_status_2 = path_2 + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_config_name_1, "Interface1/0/1") + sess.set_item(path_config_admin_status_1, "UP") + sess.set_item(path_config_name_2, "Interface1/0/2") + sess.set_item(path_config_admin_status_2, "DOWN") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "ONCE", + "updates-only": False, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": None, + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + path_name_1: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": path_name_1, + "json-data": '"Interface1/0/1"', + }, + path_config_name_1: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": path_config_name_1, + "json-data": '"Interface1/0/1"', + }, + path_config_admin_status_1: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": path_config_admin_status_1, + "json-data": '"UP"', + }, + path_name_2: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": path_name_2, + "json-data": '"Interface1/0/2"', + }, + path_config_name_2: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": path_config_name_2, + "json-data": '"Interface1/0/2"', + }, + path_config_admin_status_2: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": path_config_admin_status_2, + "json-data": '"DOWN"', + }, + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + async def test_stream_updates_only(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "STREAM", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "suppress-redundant": True, + "heartbeat-interval": 10 * 1000 * 1000 * 1000, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Update the target data. + sess.switch_datastore("running") + sess.set_item(path, "DOWN") + sess.apply_changes() + + # Wait sample interval. + time.sleep(s["sample-interval"] / 1000 / 1000 / 1000) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"DOWN"', + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + async def test_poll_updates_only(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "POLL", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": None, + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Update the target data. + sess.switch_datastore("running") + sess.set_item(path, "DOWN") + sess.apply_changes() + + # Send a poll request. + sess.rpc_send("/goldstone-telemetry:poll", {"id": params["id"]}) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"DOWN"', + }, + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + async def test_once_updates_only(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "ONCE", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": None, + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": None, + } + ], + } + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + async def test_stream_sample_not_suppress_redundant(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "STREAM", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "suppress-redundant": False, + "heartbeat-interval": 10 * 1000 * 1000 * 1000, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Wait sample interval. + time.sleep(s["sample-interval"] / 1000 / 1000 / 1000) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"UP"', + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + async def test_stream_sample_suppress_redundant(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "STREAM", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "suppress-redundant": True, + "heartbeat-interval": 10 * 1000 * 1000 * 1000, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Wait sample interval. + time.sleep(s["sample-interval"] / 1000 / 1000 / 1000) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = {} + self.assertEqual(len(self.received_notif), len(expected_notifs)) + + await self.run_test(test) + + async def test_stream_sample_suppress_redundant_but_heartbeat_expired(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "STREAM", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "suppress-redundant": True, + "heartbeat-interval": 10 * 1000 * 1000 * 1000, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Wait heartbeat interval. + time.sleep(s["heartbeat-interval"] / 1000 / 1000 / 1000) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"UP"', + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + async def test_stream_on_change_not_changed(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "STREAM", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "ON_CHANGE", + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": 10 * 1000 * 1000 * 1000, + } + ], + } + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Wait default update interval. + time.sleep(5) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = {} + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + async def test_stream_on_change_not_changed_but_heartbeat_expired(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + # Set initial data. + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Add a subscription. + params = { + "id": 1, + "mode": "STREAM", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "ON_CHANGE", + "sample-interval": None, + "suppress-redundant": None, + "heartbeat-interval": 10 * 1000 * 1000 * 1000, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Wait heartbeat interval. + time.sleep(s["heartbeat-interval"] / 1000 / 1000 / 1000) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"UP"', + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + async def test_notification_types(self): + def test(): + time.sleep(self.MOCK_WAIT) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + # Subscribe notification. + sess.subscribe_notification( + "goldstone-telemetry", + "/goldstone-telemetry:telemetry-notify-event", + self.notif_callback, + asyncio_register=False, + ) + + path_prefix = "/goldstone-interfaces:interfaces/interface[name='Interface1/0/1']" + path = path_prefix + "/config/admin-status" + + # Add a subscription. + params = { + "id": 1, + "mode": "STREAM", + "updates-only": True, + "subscriptions": [ + { + "id": 1, + "path": path, + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "suppress-redundant": True, + "heartbeat-interval": 10 * 1000 * 1000 * 1000, + } + ], + } + s = params["subscriptions"][0] + config_subscription(sess, params) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + "sync-response": { + "type": "SYNC_RESPONSE", + "request-id": params["id"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Create the target data. + sess.switch_datastore("running") + sess.set_item(path_prefix + "/config/name", "Interface1/0/1") + sess.set_item(path, "UP") + sess.apply_changes() + + # Wait sample interval. + time.sleep(s["sample-interval"] / 1000 / 1000 / 1000) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "UPDATE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + "json-data": '"UP"', + }, + } + self.assertEqual(self.received_notif, expected_notifs) + self.clear_received_notif() + + # Delete the target data. + sess.switch_datastore("running") + sess.delete_item(path_prefix) + sess.apply_changes() + + # Wait sample interval. + time.sleep(s["sample-interval"] * 2 / 1000 / 1000 / 1000) + + # Receive notifications. + time.sleep(self.NOTIFICATION_WAIT) + expected_notifs = { + s["path"]: { + "type": "DELETE", + "request-id": params["id"], + "subscription-id": s["id"], + "path": s["path"], + }, + } + self.assertEqual(self.received_notif, expected_notifs) + + await self.run_test(test) + + +if __name__ == "__main__": + unittest.main() From 9b9d5149ef5c8f625ec1afd4975bfb59ddcca1e3 Mon Sep 17 00:00:00 2001 From: Noguchi Ko <92015366+noguchiko@users.noreply.github.com> Date: Thu, 10 Nov 2022 14:17:08 +0900 Subject: [PATCH 3/6] ci: build system-telemetry This change enables the CI pipelien to build and push the streaming telemetry server container image (system-telemetry). --- .github/workflows/ci.yaml | 2 +- .github/workflows/push.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 926d1696..e72b675c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,7 +36,7 @@ jobs: platforms: linux/amd64 build-args: GS_MGMT_BUILDER_IMAGE=${{ needs.builder.outputs.tags }} targets: >- - ["north-cli", "north-snmp", "north-netconf", "north-notif", "north-gnmi", "south-sonic", "south-tai", "south-onlp", "south-system", "south-gearbox", "south-dpll", "south-netlink", "xlate-oc"] + ["north-cli", "north-snmp", "north-netconf", "north-notif", "north-gnmi", "south-sonic", "south-tai", "south-onlp", "south-system", "south-gearbox", "south-dpll", "south-netlink", "xlate-oc", "system-telemetry"] tester: if: ${{ !( ( github.event_name == 'pull_request' ) && ( github.event.pull_request.head.repo.fork == true ) ) }} uses: ./.github/workflows/build_image.yaml diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index b92ba386..774e24a2 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -42,4 +42,4 @@ jobs: file: ./docker/agent.Dockerfile build-args: GS_MGMT_BUILDER_IMAGE=${{ needs.builder.outputs.tags }} targets: >- - ["north-cli", "north-snmp", "north-netconf", "north-notif", "north-gnmi", "south-sonic", "south-tai", "south-onlp", "south-system", "south-gearbox", "south-dpll", "south-netlink", "xlate-oc"] + ["north-cli", "north-snmp", "north-netconf", "north-notif", "north-gnmi", "south-sonic", "south-tai", "south-onlp", "south-system", "south-gearbox", "south-dpll", "south-netlink", "xlate-oc", "system-telemetry"] From 1d94fb2c668479c3bd3df394645d092be369c89f Mon Sep 17 00:00:00 2001 From: Noguchi Ko <92015366+noguchiko@users.noreply.github.com> Date: Mon, 14 Nov 2022 23:08:21 +0900 Subject: [PATCH 4/6] xlate/openconfig: support openconfig-telemetry This change adds support for openconfig-telemetry model to the OpenConfig translator. Users can retrieve operational state of dynamic-subscriptions (non-persistent dial-in telemetry). --- Makefile | 2 +- scripts/gs-yang.py | 2 + src/xlate/openconfig/README.md | 3 + .../goldstone/xlate/openconfig/main.py | 4 +- .../goldstone/xlate/openconfig/telemetry.py | 129 ++++++ src/xlate/openconfig/tests/lib.py | 7 + src/xlate/openconfig/tests/test_telemetry.py | 389 ++++++++++++++++++ 7 files changed, 534 insertions(+), 2 deletions(-) create mode 100644 src/xlate/openconfig/goldstone/xlate/openconfig/telemetry.py create mode 100644 src/xlate/openconfig/tests/test_telemetry.py diff --git a/Makefile b/Makefile index 8e4ab261..b2f0c14c 100644 --- a/Makefile +++ b/Makefile @@ -137,7 +137,7 @@ unittest-dpll: unittest-openconfig: $(MAKE) clean-sysrepo - scripts/gs-yang.py --install xlate-oc south-onlp south-tai south-gearbox south-system --search-dirs yang sm/openconfig + scripts/gs-yang.py --install xlate-oc south-onlp south-tai south-gearbox south-system system-telemetry --search-dirs yang sm/openconfig cd src/xlate/openconfig && PYTHONPATH=../../lib python -m unittest -v -f $(TEST_CASE) unittest-telemetry: diff --git a/scripts/gs-yang.py b/scripts/gs-yang.py index 2dc168f8..895af1f1 100755 --- a/scripts/gs-yang.py +++ b/scripts/gs-yang.py @@ -52,6 +52,8 @@ "optical-transport/openconfig-terminal-device", "optical-transport/openconfig-transport-line-common", "optical-transport/openconfig-transport-types", + "telemetry/openconfig-telemetry", + "telemetry/openconfig-telemetry-types", ], "system-telemetry": [ "goldstone-telemetry", diff --git a/src/xlate/openconfig/README.md b/src/xlate/openconfig/README.md index 0aa62092..c2555e51 100644 --- a/src/xlate/openconfig/README.md +++ b/src/xlate/openconfig/README.md @@ -19,6 +19,8 @@ OpenConfig translator partially supports following models: - openconfig-transport-types 2021-03-22 - openconfig-types 2019-04-16 - openconfig-yang-types 2021-03-02 +- openconfig-telemetry 2018-11-21 +- openconfig-telemetry-types 2018-11-21 ## Prerequisites @@ -36,6 +38,7 @@ Other required python packages are listed in `requirements.txt`. - goldstone-platform 2019-11-01 - goldstone-system 2020-11-23 - goldstone-transponder 2019-11-01 +- goldstone-telemetry 2022-05-25 ## Install diff --git a/src/xlate/openconfig/goldstone/xlate/openconfig/main.py b/src/xlate/openconfig/goldstone/xlate/openconfig/main.py index 02323485..916e59d2 100644 --- a/src/xlate/openconfig/goldstone/xlate/openconfig/main.py +++ b/src/xlate/openconfig/goldstone/xlate/openconfig/main.py @@ -11,6 +11,7 @@ from .interfaces import InterfaceServer from .platform import PlatformServer from .terminal_device import TerminalDeviceServer +from .telemetry import TelemetryServer logger = logging.getLogger(__name__) @@ -53,7 +54,8 @@ async def _main(operational_modes): ifserver = InterfaceServer(conn) pfserver = PlatformServer(conn, operational_modes) tdserver = TerminalDeviceServer(conn, operational_modes) - servers = [ifserver, pfserver, tdserver] + tlserver = TelemetryServer(conn) + servers = [ifserver, pfserver, tdserver, tlserver] try: tasks = list( diff --git a/src/xlate/openconfig/goldstone/xlate/openconfig/telemetry.py b/src/xlate/openconfig/goldstone/xlate/openconfig/telemetry.py new file mode 100644 index 00000000..9524c27c --- /dev/null +++ b/src/xlate/openconfig/goldstone/xlate/openconfig/telemetry.py @@ -0,0 +1,129 @@ +"""OpenConfig translator for openconfig-telemetry. + +Target OpenConfig object is dynamic-subscription +("openconfig-telemetry:telemetry-system/subscriptions/dynamic-subscriptions/dynamic-subscription") for now. You can add +persistent-subscriptions and related objects. + +OpenConfig dynamic-subscription is represented as the DynamicSubscription class. +""" + + +from .lib import OpenConfigObjectFactory, OpenConfigServer + + +class DynamicSubscription: + """Represents /openconfig-telemetry:telemetry-system/subscriptions/dynamic-subscriptions/dynamic-sybscription + object. + + Args: + subscribe_request (dict): /goldstone-telemetry:subscribe-requests/subscribe-request + subscription (dict): /goldstone-telemetry:subscribe-requests/subscribe-request/subscriptions/subscription + + Attributes: + subscribe_request (dict): /goldstone-telemetry:subscribe-requests/subscribe-request + subscription (dict): /goldstone-telemetry:subscribe-requests/subscribe-request/subscriptions/subscription + data (dict): Operational state data + """ + + def __init__(self, subscribe_request, subscription): + self.subscribe_request = subscribe_request + self.subscription = subscription + self.data = { + "state": { + "protocol": "openconfig-telemetry-types:STREAM_GRPC", + "encoding": "openconfig-telemetry-types:ENC_JSON_IETF", + }, + "sensor-paths": { + "sensor-path": [], + }, + } + + def _id(self, srid, sid): + """ + Args: + srid (int): /goldstone-telemetry:subscribe-requests/subscribe-request/id + uint32 + sid (int): /goldstone-telemetry:subscribe-requests/subscribe-request/subscriptions/sunscription/id + uint32 + + Returns: + uint64: /openconfig-telemetry:telemetry-system/subscriptions/dynamic-subscriptions/dynamic-subscription/id + uint64 + """ + return (srid << 32) + sid + + def translate(self): + """Set dynamic-subscription operational state data from Goldstone operational state data.""" + id_ = self._id(self.subscribe_request["id"], self.subscription["id"]) + self.data["id"] = id_ + self.data["state"]["id"] = id_ + path = self.subscription["state"]["path"] + sensor_path = { + "path": path, + "state": { + "path": path, + }, + } + self.data["sensor-paths"]["sensor-path"].append(sensor_path) + sample_interval = self.subscription["state"].get("sample-interval") + if sample_interval is not None: + self.data["state"]["sample-interval"] = sample_interval + heartbeat_interval = self.subscription["state"].get("heartbeat-interval") + if heartbeat_interval is not None: + self.data["state"]["heartbeat-interval"] = heartbeat_interval + suppress_redundant = self.subscription["state"].get("suppress-redundant") + if suppress_redundant is not None: + self.data["state"]["suppress-redundant"] = suppress_redundant + + +class DynamicSubscriptionFactory(OpenConfigObjectFactory): + """Create OpenConfig dynamic-subscriptions from Goldstone operational state data. + + Attributes: + gs (dict): Operational state data from Goldstone native/primitive models. + """ + + def required_data(self): + return [ + { + "name": "subscribe-requests", + "xpath": "/goldstone-telemetry:subscribe-requests/subscribe-request", + "default": [], + }, + ] + + def create(self, gs): + result = [] + for subscribe_request in gs["subscribe-requests"]: + sr_state = subscribe_request.get("state") + subscriptions = None + sr_subscriptions = subscribe_request.get("subscriptions") + if sr_subscriptions is not None: + subscriptions = sr_subscriptions.get("subscription") + if subscriptions is None: + subscriptions = [] + for subscription in subscriptions: + ds = DynamicSubscription(sr_state, subscription) + ds.translate() + result.append(ds.data) + return result + + +class TelemetryServer(OpenConfigServer): + """TelemetryServer provides a service for the openconfig-telemetry module to central datastore. + + The server provides operational state information of subscriptions. + """ + + def __init__(self, conn, reconciliation_interval=10): + super().__init__(conn, "openconfig-telemetry", reconciliation_interval) + self.handlers = {"telemetry-system": {}} + self.objects = { + "telemetry-system": { + "subscriptions": { + "dynamic-subscriptions": { + "dynamic-subscription": DynamicSubscriptionFactory() + } + } + } + } diff --git a/src/xlate/openconfig/tests/lib.py b/src/xlate/openconfig/tests/lib.py index 6f775359..6d1e9ce1 100644 --- a/src/xlate/openconfig/tests/lib.py +++ b/src/xlate/openconfig/tests/lib.py @@ -131,12 +131,19 @@ def __init__(self, conn): self.handlers = {} +class MockGSTelemetryServer(MockGSServer): + def __init__(self, conn): + super().__init__(conn, "goldstone-telemetry") + self.handlers = {} + + MOCK_SERVERS = { "goldstone-interfaces": MockGSInterfaceServer, "goldstone-platform": MockGSPlatformServer, "goldstone-transponder": MockGSTransponderServer, "goldstone-system": MockGSSystemServer, "goldstone-gearbox": MockGSGearboxServer, + "goldstone-telemetry": MockGSTelemetryServer, } diff --git a/src/xlate/openconfig/tests/test_telemetry.py b/src/xlate/openconfig/tests/test_telemetry.py new file mode 100644 index 00000000..e0dcf1aa --- /dev/null +++ b/src/xlate/openconfig/tests/test_telemetry.py @@ -0,0 +1,389 @@ +"""Tests of OpenConfig translater for openconfig-telemetry.""" + +import unittest +from libyang.keyed_list import KeyedList +from goldstone.xlate.openconfig.telemetry import ( + DynamicSubscriptionFactory, + TelemetryServer, +) +from tests.lib import XlateTestCase + + +class TestDynamicSubscriptionFactory(unittest.TestCase): + """Tests for DynamicSubscriptionFactory.""" + + def test_empty_subscriptions(self): + gs_subscribe_requests = KeyedList( + [ + { + "id": 1, + "state": { + "id": 1, + "mode": "ONCE", + "updates-only": False, + }, + "subscriptions": { + "subscription": KeyedList([], "id"), + }, + }, + ], + "id", + ) + gs = {"subscribe-requests": gs_subscribe_requests} + dsf = DynamicSubscriptionFactory() + data = dsf.create(gs) + expected = [] + self.assertEqual(data, expected) + + def test_one_subscription(self): + gs_subscribe_requests = KeyedList( + [ + { + "id": 1, + "state": { + "id": 1, + "mode": "ONCE", + "updates-only": False, + }, + "subscriptions": { + "subscription": KeyedList( + [ + { + "id": 1, + "state": { + "id": 1, + "path": "/test/path", + }, + }, + ], + "id", + ), + }, + }, + ], + "id", + ) + gs = {"subscribe-requests": gs_subscribe_requests} + dsf = DynamicSubscriptionFactory() + data = dsf.create(gs) + expected = [ + { + "id": 4294967297, + "state": { + "id": 4294967297, + "protocol": "openconfig-telemetry-types:STREAM_GRPC", + "encoding": "openconfig-telemetry-types:ENC_JSON_IETF", + }, + "sensor-paths": { + "sensor-path": [ + { + "path": "/test/path", + "state": { + "path": "/test/path", + }, + }, + ], + }, + }, + ] + self.assertEqual(data, expected) + + def test_multiple_subscriptions(self): + gs_subscribe_requests = KeyedList( + [ + { + "id": 1, + "state": { + "id": 1, + "mode": "ONCE", + "updates-only": False, + }, + "subscriptions": { + "subscription": KeyedList( + [ + { + "id": 1, + "state": { + "id": 1, + "path": "/test/path", + }, + }, + ], + "id", + ), + }, + }, + { + "id": 2, + "state": { + "id": 2, + "mode": "STREAM", + "updates-only": False, + }, + "subscriptions": { + "subscription": KeyedList( + [ + { + "id": 1, + "state": { + "id": 1, + "path": "/test/path/one", + "mode": "SAMPLE", + }, + }, + { + "id": 2, + "state": { + "id": 2, + "path": "/test/path/two", + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + }, + }, + { + "id": 3, + "state": { + "id": 3, + "path": "/test/path/three", + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "heartbeat-interval": 60 * 1000 * 1000 * 1000, + }, + }, + { + "id": 4, + "state": { + "id": 4, + "path": "/test/path/four", + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "heartbeat-interval": 60 * 1000 * 1000 * 1000, + "suppress-redundant": True, + }, + }, + ], + "id", + ), + }, + }, + ], + "id", + ) + gs = {"subscribe-requests": gs_subscribe_requests} + dsf = DynamicSubscriptionFactory() + data = dsf.create(gs) + expected = [ + { + "id": 4294967296 + 1, + "state": { + "id": 4294967296 + 1, + "protocol": "openconfig-telemetry-types:STREAM_GRPC", + "encoding": "openconfig-telemetry-types:ENC_JSON_IETF", + }, + "sensor-paths": { + "sensor-path": [ + { + "path": "/test/path", + "state": { + "path": "/test/path", + }, + }, + ], + }, + }, + { + "id": 8589934592 + 1, + "state": { + "id": 8589934592 + 1, + "protocol": "openconfig-telemetry-types:STREAM_GRPC", + "encoding": "openconfig-telemetry-types:ENC_JSON_IETF", + }, + "sensor-paths": { + "sensor-path": [ + { + "path": "/test/path/one", + "state": { + "path": "/test/path/one", + }, + }, + ], + }, + }, + { + "id": 8589934592 + 2, + "state": { + "id": 8589934592 + 2, + "protocol": "openconfig-telemetry-types:STREAM_GRPC", + "encoding": "openconfig-telemetry-types:ENC_JSON_IETF", + "sample-interval": 5 * 1000 * 1000 * 1000, + }, + "sensor-paths": { + "sensor-path": [ + { + "path": "/test/path/two", + "state": { + "path": "/test/path/two", + }, + }, + ], + }, + }, + { + "id": 8589934592 + 3, + "state": { + "id": 8589934592 + 3, + "protocol": "openconfig-telemetry-types:STREAM_GRPC", + "encoding": "openconfig-telemetry-types:ENC_JSON_IETF", + "sample-interval": 5 * 1000 * 1000 * 1000, + "heartbeat-interval": 60 * 1000 * 1000 * 1000, + }, + "sensor-paths": { + "sensor-path": [ + { + "path": "/test/path/three", + "state": { + "path": "/test/path/three", + }, + }, + ], + }, + }, + { + "id": 8589934592 + 4, + "state": { + "id": 8589934592 + 4, + "protocol": "openconfig-telemetry-types:STREAM_GRPC", + "encoding": "openconfig-telemetry-types:ENC_JSON_IETF", + "sample-interval": 5 * 1000 * 1000 * 1000, + "heartbeat-interval": 60 * 1000 * 1000 * 1000, + "suppress-redundant": True, + }, + "sensor-paths": { + "sensor-path": [ + { + "path": "/test/path/four", + "state": { + "path": "/test/path/four", + }, + }, + ], + }, + }, + ] + self.assertEqual(data, expected) + + +class TestTelemetryServer(XlateTestCase): + """Tests for TelemetryServer. + + Notes: + - Mock servers take less than a second to complete the preparation. All test methods should wait a second after + calling set_mock_oper_data() to start test. + """ + + XLATE_SERVER = TelemetryServer + XLATE_SERVER_OPT = [] + XLATE_MODULES = ["openconfig-telemetry"] + MOCK_MODULES = ["goldstone-telemetry"] + + async def test_get(self): + mock_data_telemetry = { + "subscribe-requests": { + "subscribe-request": [ + { + "id": 1, + "state": { + "id": 1, + "mode": "STREAM", + "updates-only": False, + }, + "subscriptions": { + "subscription": [ + { + "id": 1, + "state": { + "id": 1, + "path": "/test/path/one", + "mode": "SAMPLE", + }, + }, + { + "id": 2, + "state": { + "id": 2, + "path": "/test/path/two", + "mode": "SAMPLE", + "sample-interval": 5 * 1000 * 1000 * 1000, + "heartbeat-interval": 60 * 1000 * 1000 * 1000, + "suppress-redundant": True, + }, + }, + ], + }, + }, + ] + } + } + self.set_mock_oper_data("goldstone-telemetry", mock_data_telemetry) + + def test(): + data = self.conn.get_operational( + "/openconfig-telemetry:telemetry-system/subscriptions/dynamic-subscriptions", + strip=False, + ) + expected = { + "telemetry-system": { + "subscriptions": { + "dynamic-subscriptions": { + "dynamic-subscription": [ + { + "id": 4294967296 + 1, + "state": { + "id": 4294967296 + 1, + "protocol": "openconfig-telemetry-types:STREAM_GRPC", + "encoding": "openconfig-telemetry-types:ENC_JSON_IETF", + }, + "sensor-paths": { + "sensor-path": [ + { + "path": "/test/path/one", + "state": { + "path": "/test/path/one", + }, + }, + ], + }, + }, + { + "id": 4294967296 + 2, + "state": { + "id": 4294967296 + 2, + "protocol": "openconfig-telemetry-types:STREAM_GRPC", + "encoding": "openconfig-telemetry-types:ENC_JSON_IETF", + "sample-interval": 5 * 1000 * 1000 * 1000, + "heartbeat-interval": 60 * 1000 * 1000 * 1000, + "suppress-redundant": True, + }, + "sensor-paths": { + "sensor-path": [ + { + "path": "/test/path/two", + "state": { + "path": "/test/path/two", + }, + }, + ], + }, + }, + ], + } + } + } + } + self.assertEqual(data, expected) + + await self.run_xlate_test(test) + + +if __name__ == "__main__": + unittest.main() From 1b29d3c06c1f150d9914949db407e56acb7088ee Mon Sep 17 00:00:00 2001 From: Noguchi Ko <92015366+noguchiko@users.noreply.github.com> Date: Mon, 14 Nov 2022 23:18:49 +0900 Subject: [PATCH 5/6] north/gnmi: support openconfig-telemetry This change adds openconfig-telemetry and openconfig-telemetry-types to supported models of the gNMI server. --- scripts/gnmi-supported-models.json | 10 ++++++++++ src/north/gnmi/tests/gnmi-supported-models.json | 10 ++++++++++ src/north/gnmi/tests/test_gnmi.py | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/scripts/gnmi-supported-models.json b/scripts/gnmi-supported-models.json index 46770c6f..2d1b9359 100644 --- a/scripts/gnmi-supported-models.json +++ b/scripts/gnmi-supported-models.json @@ -64,6 +64,16 @@ "name": "openconfig-yang-types", "organization": "OpenConfig working group", "version": "2021-03-02" + }, + { + "name": "openconfig-telemetry", + "organization": "OpenConfig working group", + "version": "2018-11-21" + }, + { + "name": "openconfig-telemetry-types", + "organization": "OpenConfig working group", + "version": "2018-11-21" } ] } \ No newline at end of file diff --git a/src/north/gnmi/tests/gnmi-supported-models.json b/src/north/gnmi/tests/gnmi-supported-models.json index 46770c6f..2d1b9359 100644 --- a/src/north/gnmi/tests/gnmi-supported-models.json +++ b/src/north/gnmi/tests/gnmi-supported-models.json @@ -64,6 +64,16 @@ "name": "openconfig-yang-types", "organization": "OpenConfig working group", "version": "2021-03-02" + }, + { + "name": "openconfig-telemetry", + "organization": "OpenConfig working group", + "version": "2018-11-21" + }, + { + "name": "openconfig-telemetry-types", + "organization": "OpenConfig working group", + "version": "2018-11-21" } ] } \ No newline at end of file diff --git a/src/north/gnmi/tests/test_gnmi.py b/src/north/gnmi/tests/test_gnmi.py index eebc66af..0f38e40d 100644 --- a/src/north/gnmi/tests/test_gnmi.py +++ b/src/north/gnmi/tests/test_gnmi.py @@ -575,6 +575,16 @@ def test_capabilities(self): "organization": "OpenConfig working group", "version": "2021-03-02", }, + { + "name": "openconfig-telemetry", + "organization": "OpenConfig working group", + "version": "2018-11-21", + }, + { + "name": "openconfig-telemetry-types", + "organization": "OpenConfig working group", + "version": "2018-11-21", + }, ], "supported_encodings": [gnmi_pb2.Encoding.JSON], "gNMI_version": "0.6.0", From b2ce056cec4008f2c4eba2edd7a5e01234000171 Mon Sep 17 00:00:00 2001 From: Noguchi Ko <92015366+noguchiko@users.noreply.github.com> Date: Wed, 8 Jun 2022 17:11:11 +0900 Subject: [PATCH 6/6] north/gnmi: add support for Subscribe RPC This change adds a support for the Subscribe RPC to the gNMI server. It uses the goldstone-telemetry model to provide streaming telemetry updates to an external telemetry collector. It allows users to synchronize the states between a device and a controller/collector. --- Makefile | 2 +- src/lib/goldstone/lib/connector/sysrepo.py | 8 + src/north/gnmi/README.md | 3 +- .../gnmi/goldstone/north/gnmi/repo/repo.py | 22 +- .../gnmi/goldstone/north/gnmi/repo/sysrepo.py | 11 +- src/north/gnmi/goldstone/north/gnmi/server.py | 320 ++- src/north/gnmi/tests/lib.py | 105 +- src/north/gnmi/tests/test_gnmi.py | 2209 +++++++++++++++++ 8 files changed, 2637 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index b2f0c14c..593e3a3a 100644 --- a/Makefile +++ b/Makefile @@ -165,6 +165,6 @@ unittest-netlink: unittest-gnmi: $(MAKE) clean-sysrepo cd src/north/gnmi && make proto - scripts/gs-yang.py --install xlate-oc --search-dirs yang sm/openconfig + scripts/gs-yang.py --install xlate-oc system-telemetry --search-dirs yang sm/openconfig cd src/north/gnmi && PYTHONPATH=../../lib python -m unittest -v -f $(TEST_CASE) cd src/north/gnmi && make clean diff --git a/src/lib/goldstone/lib/connector/sysrepo.py b/src/lib/goldstone/lib/connector/sysrepo.py index 7431706a..8d583a48 100644 --- a/src/lib/goldstone/lib/connector/sysrepo.py +++ b/src/lib/goldstone/lib/connector/sysrepo.py @@ -9,6 +9,7 @@ import sysrepo import libyang import logging +import inspect logger = logging.getLogger(__name__) @@ -127,6 +128,13 @@ def send_notification(self, name: str, notification: dict): logger.debug(f"sending notification {name}: {notification}") self.session.notification_send(name, notification) + def subscribe_notification(self, xpath, callback): + model = xpath.split("/")[1].split(":")[0] + asyncio_register = inspect.iscoroutinefunction(callback) + self.session.subscribe_notification( + model, xpath, callback, asyncio_register=asyncio_register + ) + def subscribe_notifications(self, callback): f = lambda xpath, notif_type, value, timestamp, priv: callback( {xpath: value, "eventTime": timestamp} diff --git a/src/north/gnmi/README.md b/src/north/gnmi/README.md index 12188152..e120e876 100644 --- a/src/north/gnmi/README.md +++ b/src/north/gnmi/README.md @@ -11,6 +11,7 @@ The gNMI server supports following gNMI RPCs: - `Capabilities` - `Get` - `Set` +- `Subscribe` The gNMI server supports limited `Set` transaction. It has following limitations: @@ -19,12 +20,10 @@ The gNMI server supports limited `Set` transaction. It has following limitations Currently, the gNMI server does not yet support following features: -- `Subscribe` RPC - `replace` operation for `Set` RPC - `type` specification for `Get` RPC - Wildcards in a `path` field - Value encodings other than JSON -- RPC on TLS - RPC authentication and authorization ## Prerequisites diff --git a/src/north/gnmi/goldstone/north/gnmi/repo/repo.py b/src/north/gnmi/goldstone/north/gnmi/repo/repo.py index 6ba222fc..219e56ee 100644 --- a/src/north/gnmi/goldstone/north/gnmi/repo/repo.py +++ b/src/north/gnmi/goldstone/north/gnmi/repo/repo.py @@ -8,8 +8,8 @@ def __init__(self, item): class ApplyFailedError(Exception): - def __init__(self): - super().__init__("Apply changes to the repository failed") + def __init__(self, msg): + super().__init__("Apply changes to the repository failed: {}".format(msg)) class Repository: @@ -95,3 +95,21 @@ def get_list_keys(self, path): ValueError: 'path' has an invalid value. """ pass + + def subscribe_notification(self, xpath, callback): + """Subscribe a notification. + + Args: + xpath (str): Path to the notification. + callback (func): Callback function to notify. + """ + pass + + def exec_rpc(self, xpath, params): + """Execute an RPC. + + Args: + xpath (str): Path to the RPC. + params (dict): RPC parameters. + """ + pass diff --git a/src/north/gnmi/goldstone/north/gnmi/repo/sysrepo.py b/src/north/gnmi/goldstone/north/gnmi/repo/sysrepo.py index 80df5ffb..f91c25de 100644 --- a/src/north/gnmi/goldstone/north/gnmi/repo/sysrepo.py +++ b/src/north/gnmi/goldstone/north/gnmi/repo/sysrepo.py @@ -138,8 +138,9 @@ def apply(self): self._connector.apply() except ConnectorError as e: # TODO: can split into detailed exceptions? - logger.error("apply failed. %s", e) - raise ApplyFailedError() from e + msg = f"apply failed. {e}" + logger.error(msg) + raise ApplyFailedError(msg) from e def discard(self): self._connector.discard_changes() @@ -161,3 +162,9 @@ def get_list_keys(self, path): for key in node.keys(): keys.append(key.name()) return keys + + def subscribe_notification(self, xpath, callback): + self._connector.operational_session.subscribe_notification(xpath, callback) + + def exec_rpc(self, xpath, params): + self._connector.operational_session.rpc(xpath, params) diff --git a/src/north/gnmi/goldstone/north/gnmi/server.py b/src/north/gnmi/goldstone/north/gnmi/server.py index 663f3529..9114ae9b 100644 --- a/src/north/gnmi/goldstone/north/gnmi/server.py +++ b/src/north/gnmi/goldstone/north/gnmi/server.py @@ -1,23 +1,62 @@ """gNMI server.""" +import re import logging from concurrent import futures import json import time import grpc +import random +import libyang from .proto import gnmi_pb2_grpc, gnmi_pb2 from .repo.repo import NotFoundError, ApplyFailedError logger = logging.getLogger(__name__) + GRPC_STATUS_CODE_OK = grpc.StatusCode.OK.value[0] GRPC_STATUS_CODE_UNKNOWN = grpc.StatusCode.UNKNOWN.value[0] GRPC_STATUS_CODE_INVALID_ARGUMENT = grpc.StatusCode.INVALID_ARGUMENT.value[0] GRPC_STATUS_CODE_NOT_FOUND = grpc.StatusCode.NOT_FOUND.value[0] GRPC_STATUS_CODE_ABORTED = grpc.StatusCode.ABORTED.value[0] GRPC_STATUS_CODE_UNIMPLEMENTED = grpc.StatusCode.UNIMPLEMENTED.value[0] +REGEX_PTN_LIST_KEY = re.compile(r"\[.*.*\]") + + +class InvalidArgumentError(Exception): + pass + + +def _parse_gnmi_path(gnmi_path): + xpath = "" + for elem in gnmi_path.elem: + xpath += f"/{elem.name}" + if elem.key: + for key in sorted(elem.key): + value = elem.key.get(key) + xpath += f"[{key}='{value}']" + return xpath + + +def _build_gnmi_path(xpath): + gnmi_path = gnmi_pb2.Path() + elements = list(libyang.xpath_split(xpath)) + for elem in elements: + prefix = elem[0] + name = elem[1] + if prefix is not None: + name = f"{prefix}:{name}" + keys = {} + for kv_peer in elem[2]: + keys[kv_peer[0]] = kv_peer[1] + if len(keys) > 0: + path_elem = gnmi_pb2.PathElem(name=name, key=keys) + else: + path_elem = gnmi_pb2.PathElem(name=name) + gnmi_path.elem.append(path_elem) + return gnmi_path class Request: @@ -40,23 +79,13 @@ def __init__(self, repo, prefix, gnmi_path): self.repo = repo self.prefix = prefix self.gnmi_path = gnmi_path - self.xpath = self._parse_xpath(prefix) + self._parse_xpath(gnmi_path) + self.xpath = _parse_gnmi_path(prefix) + _parse_gnmi_path(gnmi_path) logger.debug("Requested xpath: %s", self.xpath) self.status = gnmi_pb2.Error( code=GRPC_STATUS_CODE_OK, message=None, ) - def _parse_xpath(self, path): - xpath = "" - for elem in path.elem: - xpath += f"/{elem.name}" - if elem.key: - for key in sorted(elem.key): - value = elem.key.get(key) - xpath += f"[{key}='{value}']" - return xpath - def exec(self): """Execute the request. @@ -276,6 +305,169 @@ def exec(self): return +class SubscribeRequest: + """Request for gNMI Subscribe service. + + Attributes: + repo (Repository): Repository to access the datastore. + rid (int): Request ID. + subscribe (gnmi_pb2.SubscriptionList): gNMI subscribe request body. + """ + + PATH_SR = "/goldstone-telemetry:subscribe-requests/subscribe-request[id='{}']" + PATH_POLL = "/goldstone-telemetry:poll" + + SUBSCRIBE_REQUEST_MODES = { + gnmi_pb2.SubscriptionList.Mode.STREAM: "STREAM", + gnmi_pb2.SubscriptionList.Mode.ONCE: "ONCE", + gnmi_pb2.SubscriptionList.Mode.POLL: "POLL", + } + + SUBSCRIPTION_MODES = { + gnmi_pb2.SubscriptionMode.TARGET_DEFINED: "TARGET_DEFINED", + gnmi_pb2.SubscriptionMode.ON_CHANGE: "ON_CHANGE", + gnmi_pb2.SubscriptionMode.SAMPLE: "SAMPLE", + } + + def __init__(self, repo, rid, subscribe): + self._repo = repo + self._rid = rid + self._config = self._parse_config(subscribe) + self._notifs = [] + + def _parse_subscription_config(self, sid, config): + if not config.HasField("path"): + msg = "path should be specified." + logger.error(msg) + raise InvalidArgumentError(msg) + try: + mode = self.SUBSCRIPTION_MODES[config.mode] + except KeyError as e: + msg = f"mode has an invalid value {config.mode}." + logger.error(msg) + raise InvalidArgumentError(msg) from e + return { + "id": sid, + "path": _parse_gnmi_path(config.path), + "mode": mode, + "sample-interval": config.sample_interval, + "suppress-redundant": config.suppress_redundant, + "heartbeat-interval": config.heartbeat_interval, + } + + def _parse_config(self, config): + try: + mode = self.SUBSCRIBE_REQUEST_MODES[config.mode] + except KeyError as e: + raise InvalidArgumentError( + f"mode has an invalid value {config.mode}." + ) from e + subscriptions = [] + for sid, subscription in enumerate(config.subscription): + subscriptions.append(self._parse_subscription_config(sid, subscription)) + return { + "id": self._rid, + "mode": mode, + "updates-only": config.updates_only, + "subscriptions": subscriptions, + } + + def exec(self): + prefix = self.PATH_SR.format(self._rid) + configs = { + prefix + "/config/id": self._rid, + prefix + "/config/mode": self._config["mode"], + prefix + "/config/updates-only": self._config["updates-only"], + } + for s in self._config["subscriptions"]: + sid = s["id"] + sprefix = prefix + f"/subscriptions/subscription[id='{sid}']" + configs[sprefix + "/config/id"] = sid + configs[sprefix + "/config/path"] = s["path"] + if self._config["mode"] == "STREAM": + configs[sprefix + "/config/mode"] = s["mode"] + if s["sample-interval"] > 0: + configs[sprefix + "/config/sample-interval"] = s["sample-interval"] + configs[sprefix + "/config/suppress-redundant"] = s[ + "suppress-redundant" + ] + if s["heartbeat-interval"] > 0: + configs[sprefix + "/config/heartbeat-interval"] = s[ + "heartbeat-interval" + ] + with self._repo() as repo: + repo.start() + for path, value in configs.items(): + try: + repo.set(path, value) + except ValueError as e: + msg = f"failed to set {path} to the path {value}." + logger.error(msg) + raise InvalidArgumentError(msg) from e + try: + repo.apply() + except ApplyFailedError as e: + msg = f"failed to apply subscription config {self._config}. {e}." + logger.error(msg) + raise InvalidArgumentError(msg) from e + + def clear(self): + with self._repo() as repo: + repo.start() + try: + repo.delete(self.PATH_SR.format(self._rid)) + repo.apply() + except NotFoundError: + logger.info("subscription config %s to delete is not found.", self._rid) + pass + except ApplyFailedError as e: + logger.error("failed to clear subscription config %s. %s", self._rid, e) + repo.discard() + + def push_notif(self, notif): + timestamp = time.time_ns() + sr = None + if notif["type"] == "SYNC_RESPONSE": + sr = gnmi_pb2.SubscribeResponse(sync_response=True) + elif notif["type"] == "UPDATE": + sr = gnmi_pb2.SubscribeResponse( + update=gnmi_pb2.Notification( + timestamp=timestamp, + update=[ + gnmi_pb2.Update( + path=_build_gnmi_path(notif["path"]), + val=gnmi_pb2.TypedValue( + json_val=notif["json-data"].encode() + ), + ), + ], + ) + ) + elif notif["type"] == "DELETE": + sr = gnmi_pb2.SubscribeResponse( + update=gnmi_pb2.Notification( + timestamp=timestamp, + delete=[ + _build_gnmi_path(notif["path"]), + ], + ) + ) + if sr is not None: + self._notifs.insert(0, sr) + + def poll_notifs(self): + with self._repo() as repo: + repo.start() + repo.exec_rpc(self.PATH_POLL, {"id": self._rid}) + + def pull_notifs(self): + while True: + try: + yield self._notifs.pop() + except IndexError: + break + + class gNMIServicer(gnmi_pb2_grpc.gNMIServicer): """gNMIServicer provides an implementation of the methods of the gNMI service. @@ -285,11 +477,18 @@ class gNMIServicer(gnmi_pb2_grpc.gNMIServicer): """ SUPPORTED_ENCODINGS = [gnmi_pb2.Encoding.JSON] + NOTIFICATION_PULL_INTERVAL = 0.01 def __init__(self, repo, supported_models): super().__init__() self.repo = repo self.supported_models = supported_models + self._subscribe_requests = {} + self._subscribe_repo = self.repo() + self._subscribe_repo.start() + self._subscribe_repo.subscribe_notification( + "/goldstone-telemetry:telemetry-notify-event", self._notification_cb + ) def Capabilities(self, request, context): return gnmi_pb2.CapabilityResponse( @@ -471,7 +670,104 @@ def Set(self, request, context): timestamp=timestamp, ) - # TODO: Implement Subscribe(). + def _notification_cb(self, xpath, notif_type, value, timestamp, priv): + rid = value["request-id"] + try: + sr = self._subscribe_requests[rid] + except KeyError: + logger.error( + "Subscribe request %s related to the notification is not found.", rid + ) + return + sr.push_notif(value) + + def _generate_subscribe_request_id(self): + while True: + rid = random.randint(0, 0xFFFFFFFF) + if rid not in self._subscribe_requests.keys(): + return rid + + def _notify_current_states(self, sr): + sync_response = False + while True: + for notification in sr.pull_notifs(): + yield notification + if notification.sync_response: + sync_response = True + if sync_response: + break + time.sleep(self.NOTIFICATION_PULL_INTERVAL) + + def _notify_updated_states(self, sr, context): + while True: + if not context.is_active(): + break + for notification in sr.pull_notifs(): + yield notification + time.sleep(self.NOTIFICATION_PULL_INTERVAL) + + def Subscribe(self, request_iterator, context): + def set_error(code, msg): + logger.error(msg) + context.set_code(code) + context.set_details(msg) + return gnmi_pb2.Error(code=code, message=msg) + + # Create a subscription. + req = next(request_iterator) + mode = req.subscribe.mode + rid = self._generate_subscribe_request_id() + error = None + try: + sr = SubscribeRequest(self.repo, rid, req.subscribe) + self._subscribe_requests[rid] = sr + sr.exec() + except InvalidArgumentError as e: + error = set_error( + GRPC_STATUS_CODE_INVALID_ARGUMENT, + f"request has invalid argument(s). {e}", + ) + except Exception as e: + error = set_error( + GRPC_STATUS_CODE_UNKNOWN, f"an unknown error has occurred. {e}" + ) + if error is not None: + try: + self._subscribe_requests[rid].clear() + del self._subscribe_requests[rid] + except KeyError: + pass + return gnmi_pb2.SubscribeResponse(error=error) + + # Generate notifications. + try: + for notification in self._notify_current_states(sr): + yield notification + if mode == gnmi_pb2.SubscriptionList.Mode.POLL: + for req in request_iterator: + if not req.HasField("poll"): + error = set_error( + GRPC_STATUS_CODE_INVALID_ARGUMENT, + "the request is not a 'poll' request.", + ) + break + sr.poll_notifs() + for notification in self._notify_current_states(sr): + yield notification + elif mode == gnmi_pb2.SubscriptionList.Mode.STREAM: + for notification in self._notify_updated_states(sr, context): + yield notification + except Exception as e: + error = set_error( + GRPC_STATUS_CODE_UNKNOWN, f"an unknown error has occurred. {e}" + ) + finally: + self._subscribe_requests[rid].clear() + del self._subscribe_requests[rid] + if error is None: + return gnmi_pb2.SubscribeResponse() + else: + return gnmi_pb2.SubscribeResponse(error=error) def serve( diff --git a/src/north/gnmi/tests/lib.py b/src/north/gnmi/tests/lib.py index 418dd9e9..f4dd8cd2 100644 --- a/src/north/gnmi/tests/lib.py +++ b/src/north/gnmi/tests/lib.py @@ -1,5 +1,7 @@ """Tests of gNMI server.""" +# pylint: disable=W0212,C0103 + import unittest import logging import os @@ -40,8 +42,8 @@ def delete(self, xpath): raise self.exception -class MockOCPlatformServer(ServerBase): - """MockOCPlatformServer is mock handler server for openconfig-platform models. +class MockServer(ServerBase): + """MockServer is mock handler server for tests. Attributes: oper_data (dict): Data for oper_cb() to return. You can set this to configure mock's behavior. @@ -50,6 +52,8 @@ class MockOCPlatformServer(ServerBase): def __init__(self, conn, module): super().__init__(conn, module) self.oper_data = {} + self.notifs_xpath = "" + self.notifs_data = {} self.handlers = {} async def change_cb(self, event, req_id, changes, priv): @@ -58,49 +62,64 @@ async def change_cb(self, event, req_id, changes, priv): def oper_cb(self, xpath, priv): return self.oper_data + def notify(self, xpath, data): + self.conn.send_notification(xpath, data) -class MockOCInterfacesServer(ServerBase): - """MockOCInterfacesServer is mock handler server for openconfig-interfaces models. + def send_notifs(self): + for data in self.notifs_data: + self.notify(self.notifs_xpath, data) - Attributes: - oper_data (dict): Data for oper_cb() to return. You can set this to configure mock's behavior. - """ + +class MockOCPlatformServer(MockServer): + """MockOCPlatformServer is mock handler server for openconfig-platform.""" def __init__(self, conn, module): super().__init__(conn, module) + # You can customize the behavior of the mock server. self.oper_data = {} self.handlers = {} - async def change_cb(self, event, req_id, changes, priv): - pass - def oper_cb(self, xpath, priv): - return self.oper_data +class MockOCInterfacesServer(MockServer): + """MockOCInterfacesServer is mock handler server for openconfig-interfaces models.""" + def __init__(self, conn, module): + super().__init__(conn, module) + # You can customize the behavior of the mock server. + self.oper_data = {} + self.handlers = {} -class MockOCTerminalDeviceServer(ServerBase): - """MockOCTerminalDeviceServer is mock handler server for openconfig-terminal-device models. - Attributes: - oper_data (dict): Data for oper_cb() to return. You can set this to configure mock's behavior. - """ +class MockOCTerminalDeviceServer(MockServer): + """MockOCTerminalDeviceServer is mock handler server for openconfig-terminal-device .""" def __init__(self, conn, module): super().__init__(conn, module) + # You can customize the behavior of the mock server. self.oper_data = {} self.handlers = {} - async def change_cb(self, event, req_id, changes, priv): - pass - def oper_cb(self, xpath, priv): - return self.oper_data +class MockGSTelemetryServer(MockServer): + """MockGSTelemetryServer is mock handler server for goldstone-telemetry.""" + + def __init__(self, conn, module): + super().__init__(conn, module) + # You can customize the behavior of the mock server. + self.oper_data = {} + self.handlers = {} + self.poll_count = 0 + self.conn.subscribe_rpc_call("/goldstone-telemetry:poll", self.poll_cb) + + def poll_cb(self, xpath, inputs, event, priv): + self.send_notifs() MOCK_SERVERS = { "openconfig-platform": MockOCPlatformServer, "openconfig-interfaces": MockOCInterfacesServer, "openconfig-terminal-device": MockOCTerminalDeviceServer, + "goldstone-telemetry": MockGSTelemetryServer, } @@ -125,8 +144,13 @@ async def evloop(): else: if msg["type"] == "stop": return - elif msg["type"] == "set": + elif msg["type"] == "set-oper-data": servers[msg["server"]].oper_data = msg["data"] + elif msg["type"] == "set-notifs-data": + servers[msg["server"]].notifs_xpath = msg["path"] + servers[msg["server"]].notifs_data = msg["data"] + elif msg["type"] == "send-notif": + servers[msg["server"]].send_notifs() tasks.append(evloop()) tasks = [ @@ -155,11 +179,12 @@ async def asyncSetUp(self): # gNMI server to test. self._real_time = grpc_testing.strict_real_time() self.target_service = gnmi_pb2.DESCRIPTOR.services_by_name["gNMI"] - servicer = gNMIServicer(Sysrepo, load_supported_models()) - descriptors_to_services = {self.target_service: servicer} + self.servicer = gNMIServicer(Sysrepo, load_supported_models()) + descriptors_to_services = {self.target_service: self.servicer} self._real_time_server = grpc_testing.server_from_dictionary( descriptors_to_services, self._real_time ) + self.rpc = None self.conn = Connector() @@ -199,9 +224,34 @@ def set_mock_oper_data(self, server, data): server (str): Target mock server name. A key in MOCK_SERVERS. data (dict): Operational state data that the server returns. """ - self.q.put({"type": "set", "server": server, "data": data}) + self.q.put({"type": "set-oper-data", "server": server, "data": data}) + + def set_mock_notifs_data(self, server, path, data): + """Set notifications data to the mock server. + + Args: + server (str): Target mock server name. A key in MOCK_SERVERS. + path (str): Path of the notifications to send. + data (dict): Data of the notifications to send. + """ + self.q.put( + {"type": "set-notifs-data", "server": server, "path": path, "data": data} + ) + + def send_mock_notifs(self, server): + """Send notifications from the mock server. + + Args: + server (str): Target mock server name. A key in MOCK_SERVERS. + """ + self.q.put({"type": "send-notif", "server": server}) async def asyncTearDown(self): + try: + self.rpc.cancel() + except Exception: + pass + self.servicer._subscribe_repo.stop() self.tasks = [] self.conn.stop() self.q.put({"type": "stop"}) @@ -227,3 +277,10 @@ def gnmi_set(self, request): ) response, trailing_metadata, code, details = rpc.termination() return response, code + + def gnmi_subscribe(self, request): + rpc = self._real_time_server.invoke_stream_stream( + self.target_service.methods_by_name["Subscribe"], (), None + ) + rpc.send_request(request) + return rpc diff --git a/src/north/gnmi/tests/test_gnmi.py b/src/north/gnmi/tests/test_gnmi.py index 0f38e40d..eeac8d7c 100644 --- a/src/north/gnmi/tests/test_gnmi.py +++ b/src/north/gnmi/tests/test_gnmi.py @@ -1,9 +1,12 @@ """Tests of gNMI server.""" +# pylint: disable=W0212,C0103 + import unittest import time import json import grpc +import sysrepo from tests.lib import MockRepository, gNMIServerTestCase from goldstone.north.gnmi.server import ( Request, @@ -3136,5 +3139,2211 @@ def test(): await self.run_gnmi_server_test(test) +class TestSubscribe(gNMIServerTestCase): + """Tests gNMI server Subscribe Service.""" + + MOCK_MODULES = ["goldstone-telemetry"] + NOTIF_SERVER = "goldstone-telemetry" + NOTIF_PATH = "goldstone-telemetry:telemetry-notify-event" + WAIT_CREATION = 0.1 + WAIT_NOTIFICATION = 0.1 + + async def test_subscribe_stream_target_defined(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription( + path=path, mode=gnmi_pb2.SubscriptionMode.TARGET_DEFINED + ) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "STREAM", + "updates-only": False, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + "mode": "TARGET_DEFINED", + "suppress-redundant": False, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send mocked update events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "false", + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive streaming updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = False + self.assertEqual(act, expected) + + # No sync-responses. + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_stream_on_change(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription( + path=path, mode=gnmi_pb2.SubscriptionMode.ON_CHANGE + ) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "STREAM", + "updates-only": False, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + "mode": "ON_CHANGE", + "suppress-redundant": False, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send mocked update events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "false", + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive streaming updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = False + self.assertEqual(act, expected) + + # No sync-responses. + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_stream_on_change_heartbeat(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + hb_interval = 10 * 1000 * 1000 * 1000 + s1 = gnmi_pb2.Subscription( + path=path, + mode=gnmi_pb2.SubscriptionMode.ON_CHANGE, + heartbeat_interval=hb_interval, + ) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "STREAM", + "updates-only": False, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + "mode": "ON_CHANGE", + "suppress-redundant": False, + "heartbeat-interval": hb_interval, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send mocked heartbeat expired update events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive streaming updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # No sync-responses. + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_stream_sample(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s_interval = 5 * 1000 * 1000 * 1000 + s1 = gnmi_pb2.Subscription( + path=path, + mode=gnmi_pb2.SubscriptionMode.SAMPLE, + sample_interval=s_interval, + ) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "STREAM", + "updates-only": False, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + "mode": "SAMPLE", + "sample-interval": s_interval, + "suppress-redundant": False, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send mocked sampling update events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive streaming updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # No sync-responses. + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_stream_sample_suppress_redundant(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s_interval = 5 * 1000 * 1000 * 1000 + s1 = gnmi_pb2.Subscription( + path=path, + mode=gnmi_pb2.SubscriptionMode.SAMPLE, + sample_interval=s_interval, + suppress_redundant=True, + ) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "STREAM", + "updates-only": False, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + "mode": "SAMPLE", + "sample-interval": s_interval, + "suppress-redundant": True, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # No sampling update events and streaming updates because of suppress-redundant. + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_stream_sample_suppress_redundant_heartbeat(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s_interval = 5 * 1000 * 1000 * 1000 + hb_interval = 10 * 1000 * 1000 * 1000 + s1 = gnmi_pb2.Subscription( + path=path, + mode=gnmi_pb2.SubscriptionMode.SAMPLE, + sample_interval=s_interval, + suppress_redundant=True, + heartbeat_interval=hb_interval, + ) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "STREAM", + "updates-only": False, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + "mode": "SAMPLE", + "sample-interval": s_interval, + "suppress-redundant": True, + "heartbeat-interval": hb_interval, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send mocked heartbeat expired update events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive streaming updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # No sync-responses. + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_stream_updates_only(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription( + path=path, mode=gnmi_pb2.SubscriptionMode.TARGET_DEFINED + ) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + updates_only=True, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "STREAM", + "updates-only": True, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + "mode": "TARGET_DEFINED", + "suppress-redundant": False, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # No initial updates. + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send mocked update events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "false", + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive streaming updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = False + self.assertEqual(act, expected) + + # No sync-responses. + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_once(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription(path=path) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.ONCE, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "ONCE", + "updates-only": False, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Close the RPC session. + self.rpc.requests_closed() + _, code, _ = self.rpc.termination() + self.assertEqual(code, grpc.StatusCode.OK) + + # Was the subscribe-request deleted? + self.assertEqual(len(self.servicer._subscribe_requests), 0) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + with self.assertRaises(sysrepo.SysrepoNotFoundError): + sess.get_data( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_once_updates_only(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription(path=path) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.ONCE, + updates_only=True, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "ONCE", + "updates-only": True, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + notifs = [ + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # No initial updates. + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Close the RPC session. + self.rpc.requests_closed() + _, code, _ = self.rpc.termination() + self.assertEqual(code, grpc.StatusCode.OK) + + # Was the subscribe-request deleted? + self.assertEqual(len(self.servicer._subscribe_requests), 0) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + with self.assertRaises(sysrepo.SysrepoNotFoundError): + sess.get_data( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_poll(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription(path=path) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.POLL, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "POLL", + "updates-only": False, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send a poll request. + expected_time_min = time.time_ns() + poll_request = gnmi_pb2.SubscribeRequest(poll=gnmi_pb2.Poll()) + self.rpc.send_request(poll_request) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive polling updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the polling updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Close the RPC session. + self.rpc.requests_closed() + _, code, _ = self.rpc.termination() + self.assertEqual(code, grpc.StatusCode.OK) + + # Was the subscribe-request deleted? + self.assertEqual(len(self.servicer._subscribe_requests), 0) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + with self.assertRaises(sysrepo.SysrepoNotFoundError): + sess.get_data( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_poll_updates_only(self): + def test(): + # Create a Subscribe RPC session. + path_str = "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled" + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription(path=path) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.POLL, + updates_only=True, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Verify configuraion. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + expected = { + "id": generated_id, + "config": { + "id": generated_id, + "mode": "POLL", + "updates-only": True, + }, + "subscriptions": { + "subscription": [ + { + "id": 0, + "config": { + "id": 0, + "path": path_str, + }, + } + ] + }, + } + self.assertEqual(sr, expected) + + # Send mocked events. + notifs = [ + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # No initial updates. + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send a poll request. + poll_request = gnmi_pb2.SubscribeRequest(poll=gnmi_pb2.Poll()) + self.rpc.send_request(poll_request) + + time.sleep(self.WAIT_NOTIFICATION) + + # No polling updates. + + # Receive the sync-response of the polling updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Close the RPC session. + self.rpc.requests_closed() + _, code, _ = self.rpc.termination() + self.assertEqual(code, grpc.StatusCode.OK) + + # Was the subscribe-request deleted? + self.assertEqual(len(self.servicer._subscribe_requests), 0) + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + with self.assertRaises(sysrepo.SysrepoNotFoundError): + sess.get_data( + f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_a_leaf(self): + def test(): + # Create a Subscribe RPC session. + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription(path=path) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.ONCE, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Get generated request-id. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Close the RPC session. + self.rpc.requests_closed() + _, code, _ = self.rpc.termination() + self.assertEqual(code, grpc.StatusCode.OK) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_a_leaf_list(self): + def test(): + # Create a Subscribe RPC session. + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-terminal-device:terminal-device") + append_path_element(path, "logical-channels") + append_path_element(path, "channel", "index", "1") + append_path_element(path, "ingress") + append_path_element(path, "state") + append_path_element(path, "physical-channel") + s1 = gnmi_pb2.Subscription(path=path) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.ONCE, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Get generated request-id. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": ( + "/openconfig-terminal-device:terminal-device/logical-channels/channel[index='1']" + "/ingress/state/physical-channel" + ), + "json-data": "[1, 2, 3]", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = [1, 2, 3] + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Close the RPC session. + self.rpc.requests_closed() + _, code, _ = self.rpc.termination() + self.assertEqual(code, grpc.StatusCode.OK) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_a_container(self): + def test(): + # Create a Subscribe RPC session. + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + s1 = gnmi_pb2.Subscription(path=path) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.ONCE, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Get generated request-id. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/name", + "json-data": '"Interface1/0/1"', + }, + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/type", + "json-data": '"iana-if-type:ethernetCsmacd"', + }, + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/mtu", + "json-data": "1500", + }, + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + ## Verify name. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "name") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = "Interface1/0/1" + self.assertEqual(act, expected) + ## Verify type. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "type") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = "iana-if-type:ethernetCsmacd" + self.assertEqual(act, expected) + ## Verify mtu. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "mtu") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = 1500 + self.assertEqual(act, expected) + ## Verify enabled. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Close the RPC session. + self.rpc.requests_closed() + _, code, _ = self.rpc.termination() + self.assertEqual(code, grpc.StatusCode.OK) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_a_container_list(self): + def test(): + # Create a Subscribe RPC session. + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface") + s1 = gnmi_pb2.Subscription(path=path) + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.ONCE, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Get generated request-id. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/name", + "json-data": '"Interface1/0/1"', + }, + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/name", + "json-data": '"Interface1/0/1"', + }, + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/state/enabled", + "json-data": "true", + }, + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/2']/name", + "json-data": '"Interface1/0/2"', + }, + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/2']/state/name", + "json-data": '"Interface1/0/2"', + }, + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/2']/state/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + ## Verify Interface1/0/1 name. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "name") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = "Interface1/0/1" + self.assertEqual(act, expected) + ## Verify Interface1/0/1 state/name. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "name") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = "Interface1/0/1" + self.assertEqual(act, expected) + ## Verify Interface1/0/1 state/enabled. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "state") + append_path_element(path, "enabled") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + ## Verify Interface1/0/2 name. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/2") + append_path_element(path, "name") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = "Interface1/0/2" + self.assertEqual(act, expected) + ## Verify Interface1/0/2 state/name. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/2") + append_path_element(path, "state") + append_path_element(path, "name") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = "Interface1/0/2" + self.assertEqual(act, expected) + ## Verify Interface1/0/2 state/enabled. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/2") + append_path_element(path, "state") + append_path_element(path, "enabled") + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Close the RPC session. + self.rpc.requests_closed() + _, code, _ = self.rpc.termination() + self.assertEqual(code, grpc.StatusCode.OK) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_trigger_created(self): + def test(): + # Create a Subscribe RPC session. + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "config") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription(path=path, mode="ON_CHANGE") + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Get generated request-id. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # No initial updates. + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send mocked create events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/config/enabled", + "json-data": "true", + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_trigger_updated(self): + def test(): + # Create a Subscribe RPC session. + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "config") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription(path=path, mode="ON_CHANGE") + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Get generated request-id. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/config/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send mocked update events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/config/enabled", + "json-data": "false", + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = False + self.assertEqual(act, expected) + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + async def test_subscribe_trigger_deleted(self): + def test(): + # Create a Subscribe RPC session. + path = gnmi_pb2.Path() + append_path_element(path, "openconfig-interfaces:interfaces") + append_path_element(path, "interface", "name", "Interface1/0/1") + append_path_element(path, "config") + append_path_element(path, "enabled") + s1 = gnmi_pb2.Subscription(path=path, mode="ON_CHANGE") + subscriptions = [s1] + request = gnmi_pb2.SubscribeRequest( + subscribe=gnmi_pb2.SubscriptionList( + mode=gnmi_pb2.SubscriptionList.Mode.STREAM, + subscription=subscriptions, + ), + ) + self.rpc = self.gnmi_subscribe(request) + + time.sleep(self.WAIT_CREATION) + + # Get generated request-id. + with sysrepo.SysrepoConnection() as conn: + with conn.start_session() as sess: + sess.switch_datastore("running") + subscribe_requests = sess.get_data( + "/goldstone-telemetry:subscribe-requests/subscribe-request" + ) + srs = list( + subscribe_requests["subscribe-requests"]["subscribe-request"] + ) + self.assertEqual(len(srs), 1) + sr = srs[0] + generated_id = sr["id"] + + # Send mocked events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "UPDATE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/config/enabled", + "json-data": "true", + }, + { + "type": "SYNC_RESPONSE", + "request-id": generated_id, + "subscription-id": 0, + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive initial updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.update[0].path, path) + act = json.loads(actual.update.update[0].val.json_val.decode("utf-8")) + expected = True + self.assertEqual(act, expected) + + # Receive the sync-response of the initial updates. + actual = self.rpc.take_response() + self.assertEqual(actual.sync_response, True) + + # Send mocked delete events. + expected_time_min = time.time_ns() + notifs = [ + { + "type": "DELETE", + "request-id": generated_id, + "subscription-id": 0, + "path": "/openconfig-interfaces:interfaces/interface[name='Interface1/0/1']/config/enabled", + }, + ] + self.set_mock_notifs_data(self.NOTIF_SERVER, self.NOTIF_PATH, notifs) + self.send_mock_notifs(self.NOTIF_SERVER) + + time.sleep(self.WAIT_NOTIFICATION) + + # Receive updates. + actual = self.rpc.take_response() + expected_time_max = time.time_ns() + self.assertGreater(actual.update.timestamp, expected_time_min) + self.assertLess(actual.update.timestamp, expected_time_max) + self.assertEqual(actual.update.delete[0], path) + + # Close the RPC session. + self.rpc.requests_closed() + # NOTE: rpc.termination does not work for tests. We cannot close the RPC and confirm teardown procedure. + # _, code, _ = self.rpc.termination() + # self.assertEqual(code, grpc.StatusCode.OK) + # + # Was the subscribe-request deleted? + # self.assertEqual(len(self.servicer._subscribe_requests), 0) + # with sysrepo.SysrepoConnection() as conn: + # with conn.start_session() as sess: + # sess.switch_datastore("running") + # with self.assertRaises(sysrepo.SysrepoNotFoundError): + # sess.get_data( + # f"/goldstone-telemetry:subscribe-requests/subscribe-request[id='{generated_id}']" + # ) + + await self.run_gnmi_server_test(test) + + if __name__ == "__main__": unittest.main()