Streaming telemetry

.github/workflows/ci.yaml

@@ -36,7 +36,7 @@ jobs:
       platforms: linux/amd64
       build-args: GS_MGMT_BUILDER_IMAGE=${{ needs.builder.outputs.tags }}
       targets: >-
-        ["north-cli", "north-snmp", "north-netconf", "north-notif", "north-gnmi", "south-sonic", "south-tai", "south-onlp", "south-system", "south-gearbox", "south-dpll", "south-netlink", "xlate-oc"]
+        ["north-cli", "north-snmp", "north-netconf", "north-notif", "north-gnmi", "south-sonic", "south-tai", "south-onlp", "south-system", "south-gearbox", "south-dpll", "south-netlink", "xlate-oc", "system-telemetry"]
   tester:
     if: ${{ !( ( github.event_name == 'pull_request' ) && ( github.event.pull_request.head.repo.fork == true ) ) }}
     uses: ./.github/workflows/build_image.yaml

.github/workflows/push.yaml

@@ -42,4 +42,4 @@ jobs:
       file: ./docker/agent.Dockerfile
       build-args: GS_MGMT_BUILDER_IMAGE=${{ needs.builder.outputs.tags }}
       targets: >-
-        ["north-cli", "north-snmp", "north-netconf", "north-notif", "north-gnmi", "south-sonic", "south-tai", "south-onlp", "south-system", "south-gearbox", "south-dpll", "south-netlink", "xlate-oc"]
+        ["north-cli", "north-snmp", "north-netconf", "north-notif", "north-gnmi", "south-sonic", "south-tai", "south-onlp", "south-system", "south-gearbox", "south-dpll", "south-netlink", "xlate-oc", "system-telemetry"]

Makefile

@@ -37,19 +37,22 @@ TRANSPONDER_YANG ?= ./yang/goldstone-transponder.yang
 
 all: builder images tester host-packages
 
-images: south-images north-images xlate-images
+images: south-images north-images xlate-images system-images
 
 GS_SOUTH_AGENTS ?= south-sonic south-tai south-onlp south-system south-gearbox south-dpll south-netlink
 GS_NORTH_AGENTS ?= north-cli north-snmp north-netconf north-notif north-gnmi
 GS_XLATE_AGENTS ?= xlate-oc
+GS_SYSTEM_AGENTS ?= system-telemetry
 
 south-images: $(GS_SOUTH_AGENTS)
 
 north-images: $(GS_NORTH_AGENTS)
 
 xlate-images: $(GS_XLATE_AGENTS)
 
-$(GS_SOUTH_AGENTS) $(GS_NORTH_AGENTS) $(GS_XLATE_AGENTS):
+system-images: $(GS_SYSTEM_AGENTS)
+
+$(GS_SOUTH_AGENTS) $(GS_NORTH_AGENTS) $(GS_XLATE_AGENTS) $(GS_SYSTEM_AGENTS):
 	$(call build_agent_image,$@)
 
 north-snmp: snmpd
@@ -90,11 +93,11 @@ cmd:
 lint:
 	which black && exit `black -q --diff --exclude "src/north/snmp/src|src/north/gnmi/goldstone/north/gnmi/proto" src | wc -l`
 	TRANSPONDER_YANG=/tmp/test.yang $(MAKE) yang && diff /tmp/test.yang $(TRANSPONDER_YANG)
-	scripts/gs-yang.py --lint south-sonic south-onlp south-tai south-system xlate-oc --search-dirs yang sm/openconfig
-	scripts/gs-yang.py --lint south-gearbox south-onlp south-tai south-system xlate-oc --search-dirs yang sm/openconfig
+	scripts/gs-yang.py --lint south-sonic south-onlp south-tai south-system xlate-oc system-telemetry --search-dirs yang sm/openconfig
+	scripts/gs-yang.py --lint south-gearbox south-onlp south-tai south-system xlate-oc system-telemetry --search-dirs yang sm/openconfig
 	grep -rnI 'print(' src || exit 0 && exit 1
 
-unittest: unittest-lib unittest-cli unittest-gearbox unittest-dpll unittest-openconfig unittest-tai unittest-sonic unittest-gnmi
+unittest: unittest-lib unittest-cli unittest-gearbox unittest-dpll unittest-openconfig unittest-tai unittest-sonic unittest-gnmi unittest-telemetry
 
 rust-unittest: unittest-netlink
 
@@ -134,9 +137,14 @@ unittest-dpll:
 
 unittest-openconfig:
 	$(MAKE) clean-sysrepo
-	scripts/gs-yang.py --install xlate-oc south-onlp south-tai south-gearbox south-system --search-dirs yang sm/openconfig
+	scripts/gs-yang.py --install xlate-oc south-onlp south-tai south-gearbox south-system system-telemetry --search-dirs yang sm/openconfig
 	cd src/xlate/openconfig && PYTHONPATH=../../lib python -m unittest -v -f $(TEST_CASE)
 
+unittest-telemetry:
+	$(MAKE) clean-sysrepo
+	scripts/gs-yang.py --install system-telemetry south-gearbox --search-dirs yang sm/openconfig
+	cd src/system/telemetry && PYTHONPATH=../../lib python -m unittest -v -f $(TEST_CASE)
+
 unittest-tai:
 	$(MAKE) clean-sysrepo
 	scripts/gs-yang.py --install south-tai --search-dirs yang
@@ -157,6 +165,6 @@ unittest-netlink:
 unittest-gnmi:
 	$(MAKE) clean-sysrepo
 	cd src/north/gnmi && make proto
-	scripts/gs-yang.py --install xlate-oc --search-dirs yang sm/openconfig
+	scripts/gs-yang.py --install xlate-oc system-telemetry --search-dirs yang sm/openconfig
 	cd src/north/gnmi && PYTHONPATH=../../lib python -m unittest -v -f $(TEST_CASE)
-	cd src/north/gnmi && make clean
+	cd src/north/gnmi && make clean

README.md

@@ -12,6 +12,8 @@ The components in this repo are pre-installed in Goldstone NOS.
     - e.g) Switch ASIC, Transponder, Gearbox, Peripheral devices(Thermal sensors, LED, fan etc..)
 - Goldstone management translation daemons that translate standarized YANG models and Goldstone YANG models (Red boxes in the following diagram)
     - OpenConfig Translator
+- Goldstone management system daemons that provide service to north daemons by only interacting with south or xlate daemons
+    - Streaming telemetry Server
 - Goldstone YANG models
     - The schemas that are used between north and south daemons
 
@@ -85,7 +87,7 @@ The intention to have native YANG models is to fully cover what the underneath h
 
 Using the standard YANG models ([OpenConfig](https://www.openconfig.net/), [OpenROADM](http://openroadm.org/) etc..) is also supported by using translater daemons.
 
-`goldstone-mgmt` framework has three kinds of daemon which interact with sysrepo datastore.
+`goldstone-mgmt` framework has four kinds of daemon which interact with sysrepo datastore.
 
 - north daemon
     - provides northbound API (CLI, NETCONF, SNMP, RESTCONF, gNMI etc..)
@@ -97,6 +99,10 @@ Using the standard YANG models ([OpenConfig](https://www.openconfig.net/), [Open
 - translation daemon
     - translator of the standarized YANG models and Goldstone YANG models
     - source code under [`src/xlate`](https://github.com/oopt-goldstone/goldstone-mgmt/tree/master/src/xlate)
+- system daemon
+    - provides system utility services for north daemons
+    - optionally uses native YANG models to interact with sysrepo
+    - source code under [`src/system`](https://github.com/oopt-goldstone/goldstone-mgmt/tree/master/src/system)
 
 ### How to build
 

docker/agent.Dockerfile

@@ -278,6 +278,14 @@ RUN --mount=type=bind,source=src/xlate/openconfig,target=/src,rw pip install /sr
 RUN mkdir -p /current
 RUN --mount=type=bind,source=scripts/,target=/scripts,rw cp /scripts/operational-modes.json /current
 
+#---
+# system-telemetry
+#---
+
+FROM base AS system-telemetry
+
+RUN --mount=type=bind,source=src/system/telemetry,target=/src,rw pip install /src
+
 #---
 # default image
 #---

k8s/prep.yaml

@@ -47,7 +47,7 @@ spec:
             - name: prep-sysrepo
               image: ghcr.io/oopt-goldstone/mgmt/north-cli:latest
               imagePullPolicy: IfNotPresent
-              command: ["gs-yang.py", "--install", "south-onlp", "south-sonic", "south-tai", "south-system", "xlate-oc"]
+              command: ["gs-yang.py", "--install", "south-onlp", "south-sonic", "south-tai", "south-system", "xlate-oc", "system-telemetry"]
               volumeMounts:
               - name: shm
                 mountPath: /dev/shm

k8s/system-telemetry.yaml

@@ -0,0 +1,81 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+    name: system-telemetry
+    labels:
+        gs-mgmt: system-telemetry
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+    name: system-telemetry
+    labels:
+        gs-mgmt: system-telemetry
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: view
+subjects:
+- kind: ServiceAccount
+  name: system-telemetry
+  namespace: default
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+    name: system-telemetry
+    labels:
+        app: gs-mgmt
+        gs-mgmt: system-telemetry
+spec:
+    selector:
+        matchLabels:
+            app: system-telemetry
+    template:
+        metadata:
+            labels:
+                app: system-telemetry
+        spec:
+            serviceAccountName: system-telemetry
+            initContainers:
+            - name: wait-prep
+              image: docker.io/lachlanevenson/k8s-kubectl:latest
+              imagePullPolicy: IfNotPresent
+              command: ['kubectl', 'wait', '--for=condition=complete', 'job/prep-gs-mgmt']
+            containers:
+            - name: system-telemetry
+              image: ghcr.io/oopt-goldstone/mgmt/system-telemetry:latest
+              imagePullPolicy: IfNotPresent
+              command: ['gssystemd-telemetry']
+              args: ['--verbose']
+              volumeMounts:
+              - name: shm
+                mountPath: /dev/shm
+              - name: sysrepo
+                mountPath: /var/lib/sysrepo
+              livenessProbe:
+                httpGet:
+                  path: /healthz
+                  port: liveness-port
+                failureThreshold: 10
+                periodSeconds: 5
+                timeoutSeconds: 5
+              startupProbe:
+                httpGet:
+                  path: /healthz
+                  port: liveness-port
+                failureThreshold: 30
+                periodSeconds: 10
+              ports:
+              - name: liveness-port
+                containerPort: 8080
+            volumes:
+            - name: shm
+              hostPath:
+                  path: /dev/shm
+            - name: sysrepo
+              hostPath:
+                  path: /var/lib/sysrepo

scripts/gnmi-supported-models.json

@@ -64,6 +64,16 @@
             "name": "openconfig-yang-types",
             "organization": "OpenConfig working group",
             "version": "2021-03-02"
+        },
+        {
+            "name": "openconfig-telemetry",
+            "organization": "OpenConfig working group",
+            "version": "2018-11-21"
+        },
+        {
+            "name": "openconfig-telemetry-types",
+            "organization": "OpenConfig working group",
+            "version": "2018-11-21"
         }
     ]
 }

scripts/gs-yang.py

@@ -52,6 +52,11 @@
         "optical-transport/openconfig-terminal-device",
         "optical-transport/openconfig-transport-line-common",
         "optical-transport/openconfig-transport-types",
+        "telemetry/openconfig-telemetry",
+        "telemetry/openconfig-telemetry-types",
+    ],
+    "system-telemetry": [
+        "goldstone-telemetry",
     ],
 }
 

src/lib/goldstone/lib/connector/sysrepo.py

@@ -9,6 +9,7 @@
 import sysrepo
 import libyang
 import logging
+import inspect
 
 logger = logging.getLogger(__name__)
 
@@ -127,6 +128,13 @@ def send_notification(self, name: str, notification: dict):
         logger.debug(f"sending notification {name}: {notification}")
         self.session.notification_send(name, notification)
 
+    def subscribe_notification(self, xpath, callback):
+        model = xpath.split("/")[1].split(":")[0]
+        asyncio_register = inspect.iscoroutinefunction(callback)
+        self.session.subscribe_notification(
+            model, xpath, callback, asyncio_register=asyncio_register
+        )
+
     def subscribe_notifications(self, callback):
         f = lambda xpath, notif_type, value, timestamp, priv: callback(
             {xpath: value, "eventTime": timestamp}

src/north/gnmi/README.md

@@ -11,6 +11,7 @@ The gNMI server supports following gNMI RPCs:
 - `Capabilities`
 - `Get`
 - `Set`
+- `Subscribe`
 
 The gNMI server supports limited `Set` transaction. It has following limitations:
 
@@ -19,12 +20,10 @@ The gNMI server supports limited `Set` transaction. It has following limitations
 
 Currently, the gNMI server does not yet support following features:
 
-- `Subscribe` RPC
 - `replace` operation for `Set` RPC
 - `type` specification for `Get` RPC
 - Wildcards in a `path` field
 - Value encodings other than JSON
-- RPC on TLS
 - RPC authentication and authorization
 
 ## Prerequisites

src/north/gnmi/goldstone/north/gnmi/repo/repo.py

@@ -8,8 +8,8 @@ def __init__(self, item):
 
 
 class ApplyFailedError(Exception):
-    def __init__(self):
-        super().__init__("Apply changes to the repository failed")
+    def __init__(self, msg):
+        super().__init__("Apply changes to the repository failed: {}".format(msg))
 
 
 class Repository:
@@ -95,3 +95,21 @@ def get_list_keys(self, path):
             ValueError: 'path' has an invalid value.
         """
         pass
+
+    def subscribe_notification(self, xpath, callback):
+        """Subscribe a notification.
+
+        Args:
+            xpath (str): Path to the notification.
+            callback (func): Callback function to notify.
+        """
+        pass
+
+    def exec_rpc(self, xpath, params):
+        """Execute an RPC.
+
+        Args:
+            xpath (str): Path to the RPC.
+            params (dict): RPC parameters.
+        """
+        pass

src/north/gnmi/goldstone/north/gnmi/repo/sysrepo.py

@@ -138,8 +138,9 @@ def apply(self):
             self._connector.apply()
         except ConnectorError as e:
             # TODO: can split into detailed exceptions?
-            logger.error("apply failed. %s", e)
-            raise ApplyFailedError() from e
+            msg = f"apply failed. {e}"
+            logger.error(msg)
+            raise ApplyFailedError(msg) from e
 
     def discard(self):
         self._connector.discard_changes()
@@ -161,3 +162,9 @@ def get_list_keys(self, path):
         for key in node.keys():
             keys.append(key.name())
         return keys
+
+    def subscribe_notification(self, xpath, callback):
+        self._connector.operational_session.subscribe_notification(xpath, callback)
+
+    def exec_rpc(self, xpath, params):
+        self._connector.operational_session.rpc(xpath, params)