From 83b7b28018592396ce3f26d4468fceef137205da Mon Sep 17 00:00:00 2001 From: davidkneipp Date: Tue, 2 Jan 2024 13:42:46 +1000 Subject: [PATCH 1/8] Add total counters for diameter requests and responses --- services/diameterService.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/services/diameterService.py b/services/diameterService.py index 04629b3..962d8fd 100644 --- a/services/diameterService.py +++ b/services/diameterService.py @@ -31,6 +31,7 @@ def __init__(self): self.redisReaderMessaging = RedisMessagingAsync(host=self.redisHost, port=self.redisPort, useUnixSocket=self.redisUseUnixSocket, unixSocketPath=self.redisUnixSocketPath) self.redisWriterMessaging = RedisMessagingAsync(host=self.redisHost, port=self.redisPort, useUnixSocket=self.redisUseUnixSocket, unixSocketPath=self.redisUnixSocketPath) self.redisPeerMessaging = RedisMessagingAsync(host=self.redisHost, port=self.redisPort, useUnixSocket=self.redisUseUnixSocket, unixSocketPath=self.redisUnixSocketPath) + self.redisMetricMessaging = RedisMessagingAsync(host=self.redisHost, port=self.redisPort, useUnixSocket=self.redisUseUnixSocket, unixSocketPath=self.redisUnixSocketPath) self.banners = Banners() self.logTool = LogTool(config=self.config) self.diameterLibrary = DiameterAsync(logTool=self.logTool) @@ -115,6 +116,14 @@ async def logProcessedMessages(self): while True: await(self.logTool.logAsync(service='Diameter', level='info', message=f"[Diameter] [logProcessedMessages] Processed {self.diameterRequests} inbound diameter messages in the last {self.benchmarkingInterval} second(s)")) await(self.logTool.logAsync(service='Diameter', level='info', message=f"[Diameter] [logProcessedMessages] Processed {self.diameterResponses} outbound in the last {self.benchmarkingInterval} second(s)")) + await(self.redisMetricMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_request_count', + metricType='counter', metricAction='inc', + metricValue=float(self.diameterRequests), metricHelp='Number of Diameter Requests Received', + metricExpiry=60)) + await(self.redisMetricMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count', + metricType='counter', metricAction='inc', + metricValue=float(self.diameterResponses), metricHelp='Number of Diameter Responses Sent', + metricExpiry=60)) self.diameterRequests = 0 self.diameterResponses = 0 await(asyncio.sleep(benchmarkInterval)) From 8d3549fe8f473489d4f39e6727e32c14b960eeef Mon Sep 17 00:00:00 2001 From: davidkneipp Date: Tue, 2 Jan 2024 15:21:35 +1000 Subject: [PATCH 2/8] Metrics on a per-host basis --- services/diameterService.py | 9 ++++++--- services/hssService.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/services/diameterService.py b/services/diameterService.py index 962d8fd..6f777b0 100644 --- a/services/diameterService.py +++ b/services/diameterService.py @@ -117,11 +117,14 @@ async def logProcessedMessages(self): await(self.logTool.logAsync(service='Diameter', level='info', message=f"[Diameter] [logProcessedMessages] Processed {self.diameterRequests} inbound diameter messages in the last {self.benchmarkingInterval} second(s)")) await(self.logTool.logAsync(service='Diameter', level='info', message=f"[Diameter] [logProcessedMessages] Processed {self.diameterResponses} outbound in the last {self.benchmarkingInterval} second(s)")) await(self.redisMetricMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_request_count', - metricType='counter', metricAction='inc', - metricValue=float(self.diameterRequests), metricHelp='Number of Diameter Requests Received', + metricType='gauge', metricAction='set', + metricValue=float(self.diameterRequests), + metricLabels={'benchmark_interval': self.benchmarkingInterval}, + metricHelp='Number of Diameter Requests Received', metricExpiry=60)) await(self.redisMetricMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count', - metricType='counter', metricAction='inc', + metricType='gauge', metricAction='set', + metricLabels={'benchmark_interval': self.benchmarkingInterval}, metricValue=float(self.diameterResponses), metricHelp='Number of Diameter Responses Sent', metricExpiry=60)) self.diameterRequests = 0 diff --git a/services/hssService.py b/services/hssService.py index 46abcbc..8926de1 100644 --- a/services/hssService.py +++ b/services/hssService.py @@ -56,6 +56,21 @@ def handleQueue(self): inboundPort = inboundMessage.get('clientPort', None) inboundTimestamp = inboundMessage.get('inbound-received-timestamp', None) + try: + diameterPeers = json.loads(self.redisMessaging.getValue("ActiveDiameterPeers")) + + for diameterPeer in diameterPeers: + if diameterPeer.get('ipAddress', '') == inboundHost and str(diameterPeer.get('port', '')) == str(inboundPort): + self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_request_count_host', + metricType='gauge', metricAction='inc', + metricLabels={ + "host": diameterPeer['diameterHostname']}, + metricValue=float(1), metricHelp='Number of Diameter Requests Recieved per Host', + metricExpiry=60) + + except Exception as e: + pass + try: diameterOutbound = self.diameterLibrary.generateDiameterResponse(binaryData=inboundBinary) @@ -90,6 +105,22 @@ def handleQueue(self): if self.benchmarking: self.logTool.log(service='HSS', level='info', message=f"[HSS] [handleQueue] [{diameterMessageTypeInbound}] Time taken to process request: {round(((time.perf_counter() - startTime)*1000), 3)} ms", redisClient=self.redisMessaging) + try: + diameterPeers = json.loads(self.redisMessaging.getValue("ActiveDiameterPeers")) + + for diameterPeer in diameterPeers: + if diameterPeer.get('ipAddress', '') == inboundHost and str(diameterPeer.get('port', '')) == str(inboundPort): + self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count_host', + metricType='gauge', metricAction='inc', + metricLabels={ + "host": diameterPeer['diameterHostname']}, + metricValue=float(1), metricHelp='Number of Diameter Responses Sent per Host', + metricExpiry=60) + + except Exception as e: + pass + + except Exception as e: self.logTool.log(service='HSS', level='error', message=f"[HSS] [handleQueue] Exception: {traceback.format_exc()}", redisClient=self.redisMessaging) continue From 705a4b39fb5cf25905e3501a16142776419e9ad5 Mon Sep 17 00:00:00 2001 From: davidkneipp Date: Tue, 2 Jan 2024 17:18:02 +1000 Subject: [PATCH 3/8] dictionary fix --- services/hssService.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/hssService.py b/services/hssService.py index 8926de1..00b0bac 100644 --- a/services/hssService.py +++ b/services/hssService.py @@ -60,11 +60,11 @@ def handleQueue(self): diameterPeers = json.loads(self.redisMessaging.getValue("ActiveDiameterPeers")) for diameterPeer in diameterPeers: - if diameterPeer.get('ipAddress', '') == inboundHost and str(diameterPeer.get('port', '')) == str(inboundPort): + if diameterPeers[diameterPeer].get('ipAddress', '') == inboundHost and diameterPeers[diameterPeer].get('port', '') == inboundPort: self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_request_count_host', metricType='gauge', metricAction='inc', metricLabels={ - "host": diameterPeer['diameterHostname']}, + "host": diameterPeers[diameterPeer]['diameterHostname']}, metricValue=float(1), metricHelp='Number of Diameter Requests Recieved per Host', metricExpiry=60) @@ -109,11 +109,11 @@ def handleQueue(self): diameterPeers = json.loads(self.redisMessaging.getValue("ActiveDiameterPeers")) for diameterPeer in diameterPeers: - if diameterPeer.get('ipAddress', '') == inboundHost and str(diameterPeer.get('port', '')) == str(inboundPort): + if diameterPeers[diameterPeer].get('ipAddress', '') == inboundHost and diameterPeers[diameterPeer].get('port', '') == inboundPort: self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count_host', metricType='gauge', metricAction='inc', metricLabels={ - "host": diameterPeer['diameterHostname']}, + "host": diameterPeers[diameterPeer]['diameterHostname']}, metricValue=float(1), metricHelp='Number of Diameter Responses Sent per Host', metricExpiry=60) From 0fb22a216956d5f597b382851343998e43474467 Mon Sep 17 00:00:00 2001 From: davidkneipp Date: Wed, 3 Jan 2024 09:57:39 +1000 Subject: [PATCH 4/8] Add diameter_application_id and diameter_cmd_code labels to diameter responses --- lib/diameter.py | 10 +++++++++- services/diameterService.py | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/diameter.py b/lib/diameter.py index 3df733c..883e82b 100644 --- a/lib/diameter.py +++ b/lib/diameter.py @@ -772,12 +772,20 @@ def generateDiameterResponse(self, binaryData: str) -> str: self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count_successful', metricType='counter', metricAction='inc', + metricLabels={ + "diameter_application_id": packet_vars["ApplicationId"], + "diameter_cmd_code": packet_vars["command_code"], + }, metricValue=1.0, metricHelp='Number of Successful Diameter Responses', metricExpiry=60) return response except Exception as e: self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count_fail', - metricType='counter', metricAction='inc', + metricType='counter', metricAction='inc', + metricLabels={ + "diameter_application_id": packet_vars["ApplicationId"], + "diameter_cmd_code": packet_vars["command_code"], + }, metricValue=1.0, metricHelp='Number of Failed Diameter Responses', metricExpiry=60) return '' diff --git a/services/diameterService.py b/services/diameterService.py index 6f777b0..87bb9aa 100644 --- a/services/diameterService.py +++ b/services/diameterService.py @@ -117,13 +117,13 @@ async def logProcessedMessages(self): await(self.logTool.logAsync(service='Diameter', level='info', message=f"[Diameter] [logProcessedMessages] Processed {self.diameterRequests} inbound diameter messages in the last {self.benchmarkingInterval} second(s)")) await(self.logTool.logAsync(service='Diameter', level='info', message=f"[Diameter] [logProcessedMessages] Processed {self.diameterResponses} outbound in the last {self.benchmarkingInterval} second(s)")) await(self.redisMetricMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_request_count', - metricType='gauge', metricAction='set', + metricType='gauge', metricAction='inc', metricValue=float(self.diameterRequests), metricLabels={'benchmark_interval': self.benchmarkingInterval}, metricHelp='Number of Diameter Requests Received', metricExpiry=60)) await(self.redisMetricMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count', - metricType='gauge', metricAction='set', + metricType='gauge', metricAction='inc', metricLabels={'benchmark_interval': self.benchmarkingInterval}, metricValue=float(self.diameterResponses), metricHelp='Number of Diameter Responses Sent', metricExpiry=60)) From bb9d918d481aa0ccdcab657682236cd175990028 Mon Sep 17 00:00:00 2001 From: davidkneipp Date: Wed, 3 Jan 2024 10:13:37 +1000 Subject: [PATCH 5/8] Counter for diameter requests by application ID --- lib/diameter.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/diameter.py b/lib/diameter.py index 883e82b..ea623e0 100644 --- a/lib/diameter.py +++ b/lib/diameter.py @@ -757,6 +757,15 @@ def generateDiameterResponse(self, binaryData: str) -> str: self.logTool.log(service='HSS', level='debug', message=packet_vars, redisClient=self.redisMessaging) return + self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_request_count_application_id', + metricType='counter', metricAction='inc', + metricLabels={ + "diameter_application_id": packet_vars["ApplicationId"], + "diameter_cmd_code": packet_vars["command_code"], + }, + metricValue=1.0, metricHelp='Number of Diameter Requests by Application Id', + metricExpiry=60) + for diameterApplication in self.diameterResponseList: try: assert(packet_vars["command_code"] == diameterApplication["commandCode"]) From 51045052b53f371b3521e3b86bf5d9d00d00a2e2 Mon Sep 17 00:00:00 2001 From: davidkneipp Date: Wed, 3 Jan 2024 10:16:59 +1000 Subject: [PATCH 6/8] Counter name update --- lib/diameter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/diameter.py b/lib/diameter.py index ea623e0..1f3ee07 100644 --- a/lib/diameter.py +++ b/lib/diameter.py @@ -779,7 +779,7 @@ def generateDiameterResponse(self, binaryData: str) -> str: except Exception as e: continue - self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count_successful', + self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count_application_id_successful', metricType='counter', metricAction='inc', metricLabels={ "diameter_application_id": packet_vars["ApplicationId"], @@ -789,7 +789,7 @@ def generateDiameterResponse(self, binaryData: str) -> str: metricExpiry=60) return response except Exception as e: - self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count_fail', + self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count_application_id_fail', metricType='counter', metricAction='inc', metricLabels={ "diameter_application_id": packet_vars["ApplicationId"], From 3bd5d4f9321599a43c34889ce7a77ab5ec90dc94 Mon Sep 17 00:00:00 2001 From: davidkneipp Date: Tue, 23 Jan 2024 13:42:21 +1000 Subject: [PATCH 7/8] Fix metrics, add prom_diam_connected_state --- CHANGELOG.md | 1 + lib/CryptoTool.py | 0 lib/__init__.py | 0 lib/banners.py | 0 lib/database.py | 10 +++- lib/diameter.py | 80 +++++++++++++++++++++++++------- lib/diameterAsync.py | 0 lib/logtool.py | 0 lib/lte.py | 0 lib/messaging.py | 0 lib/messagingAsync.py | 0 lib/metrics.py | 0 lib/milenage.py | 0 lib/mongodb_insert_subscriber.py | 0 services/apiService.py | 35 +++++++++++--- services/diameterService.py | 48 ++++++++++++++++--- services/georedService.py | 60 +++++++++++++++++++----- services/hssService.py | 16 +++++-- services/logService.py | 0 services/metricService.py | 2 - 20 files changed, 202 insertions(+), 50 deletions(-) mode change 100644 => 100755 lib/CryptoTool.py mode change 100644 => 100755 lib/__init__.py mode change 100644 => 100755 lib/banners.py mode change 100644 => 100755 lib/diameter.py mode change 100644 => 100755 lib/diameterAsync.py mode change 100644 => 100755 lib/logtool.py mode change 100644 => 100755 lib/lte.py mode change 100644 => 100755 lib/messaging.py mode change 100644 => 100755 lib/messagingAsync.py mode change 100644 => 100755 lib/metrics.py mode change 100644 => 100755 lib/milenage.py mode change 100644 => 100755 lib/mongodb_insert_subscriber.py mode change 100644 => 100755 services/apiService.py mode change 100644 => 100755 services/diameterService.py mode change 100644 => 100755 services/georedService.py mode change 100644 => 100755 services/hssService.py mode change 100644 => 100755 services/logService.py mode change 100644 => 100755 services/metricService.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4799da4..16fbacc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Roaming management on a per-subscriber basis, through subscriber.roaming_enabled and subscriber.roaming_rule_list. - Support for Gx and Rx auth of unknown subscribers attaching via SOS. - Preliminary support for SCTP. +- Additional prometheus metrics. ## [1.0.0] - 2023-09-27 diff --git a/lib/CryptoTool.py b/lib/CryptoTool.py old mode 100644 new mode 100755 diff --git a/lib/__init__.py b/lib/__init__.py old mode 100644 new mode 100755 diff --git a/lib/banners.py b/lib/banners.py old mode 100644 new mode 100755 diff --git a/lib/database.py b/lib/database.py index 7770118..8b19f66 100755 --- a/lib/database.py +++ b/lib/database.py @@ -2381,7 +2381,10 @@ def Store_IMSI_IMEI_Binding(self, imsi, imei, match_response_code, propagate=Tru metricLabels={'imei_prefix': device_info['tacPrefix'], 'device_type': device_info['name'], 'device_name': device_info['model']}, - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') except Exception as E: self.logTool.log(service='Database', level='debug', message="Failed to get device info from TAC", redisClient=self.redisMessaging) self.redisMessaging.sendMetric(serviceName='database', metricName='prom_eir_devices', @@ -2390,7 +2393,10 @@ def Store_IMSI_IMEI_Binding(self, imsi, imei, match_response_code, propagate=Tru metricLabels={'imei_prefix': str(imei)[0:8], 'device_type': 'Unknown', 'device_name': 'Unknown'}, - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') else: self.logTool.log(service='Database', level='debug', message="No TAC database configured, skipping device info lookup", redisClient=self.redisMessaging) diff --git a/lib/diameter.py b/lib/diameter.py old mode 100644 new mode 100755 index 6646478..75e83ee --- a/lib/diameter.py +++ b/lib/diameter.py @@ -802,7 +802,10 @@ def generateDiameterResponse(self, binaryData: str) -> str: "diameter_cmd_code": packet_vars["command_code"], }, metricValue=1.0, metricHelp='Number of Diameter Requests by Application Id', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') for diameterApplication in self.diameterResponseList: try: @@ -828,7 +831,10 @@ def generateDiameterResponse(self, binaryData: str) -> str: "diameter_cmd_code": packet_vars["command_code"], }, metricValue=1.0, metricHelp='Number of Successful Diameter Responses', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') return response except Exception as e: self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count_application_id_fail', @@ -838,7 +844,10 @@ def generateDiameterResponse(self, binaryData: str) -> str: "diameter_cmd_code": packet_vars["command_code"], }, metricValue=1.0, metricHelp='Number of Failed Diameter Responses', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') return '' def generateDiameterRequest(self, requestType: str, **kwargs) -> str: @@ -1212,15 +1221,24 @@ def Generate_Prom_Stats(self): self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_ims_subs', metricType='gauge', metricAction='set', metricValue=len(self.database.Get_Served_IMS_Subscribers(get_local_users_only=True)), metricHelp='Number of attached IMS Subscribers', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_mme_subs', metricType='gauge', metricAction='set', metricValue=len(self.database.Get_Served_Subscribers(get_local_users_only=True)), metricHelp='Number of attached MME Subscribers', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') self.redisMessaging.sendMetric(serviceName='diameter', metricName='prom_pcrf_subs', metricType='gauge', metricAction='set', metricValue=len(self.database.Get_Served_PCRF_Subscribers(get_local_users_only=True)), metricHelp='Number of attached PCRF Subscribers', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') except Exception as e: self.logTool.log(service='HSS', level='debug', message="Failed to generate Prometheus Stats for IMS Subscribers", redisClient=self.redisMessaging) self.logTool.log(service='HSS', level='debug', message=e, redisClient=self.redisMessaging) @@ -1624,7 +1642,10 @@ def Answer_16777251_318(self, packet_vars, avps): "event": "Disabled User", "imsi_prefix": str(imsi[0:6])}, metricHelp='Diameter Authentication related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') session_id = self.get_avp_data(avps, 263)[0] #Get Session-ID avp += self.generate_avp(263, 40, session_id) #Session-ID AVP set avp += self.generate_avp(264, 40, self.OriginHost) #Origin Host @@ -1654,7 +1675,10 @@ def Answer_16777251_318(self, packet_vars, avps): "event": "Unknown User", "imsi_prefix": str(imsi[0:6])}, metricHelp='Diameter Authentication related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') #Handle if the subscriber is not present in HSS return "DIAMETER_ERROR_USER_UNKNOWN" self.logTool.log(service='HSS', level='debug', message="Subscriber " + str(imsi) + " is unknown in database", redisClient=self.redisMessaging) avp = '' @@ -1736,7 +1760,10 @@ def Answer_16777251_318(self, packet_vars, avps): "event": "Resync", "imsi_prefix": str(imsi[0:6])}, metricHelp='Diameter Authentication related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') auts = str(sub_avp['misc_data'])[32:] rand = str(sub_avp['misc_data'])[:32] rand = binascii.unhexlify(rand) @@ -2188,7 +2215,10 @@ def Answer_16777238_272(self, packet_vars, avps): "event": "Unknown User", "imsi_prefix": str(imsi[0:6])}, metricHelp='Diameter Authentication related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') avp += self.generate_avp(268, 40, self.int_to_hex(5030, 4)) #Result Code (DIAMETER ERROR - User Unknown) response = self.generate_diameter_packet("01", "40", 272, 16777238, packet_vars['hop-by-hop-identifier'], packet_vars['end-to-end-identifier'], avp) #Generate Diameter packet return response @@ -2238,7 +2268,10 @@ def Answer_16777216_300(self, packet_vars, avps): "event": "Unknown User", "imsi_prefix": str(imsi[0:6])}, metricHelp='Diameter Authentication related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') result_code = 5001 #IMS User Unknown #Experimental Result AVP avp_experimental_result = '' @@ -2422,7 +2455,10 @@ def Answer_16777216_302(self, packet_vars, avps): "event": "Unknown User", "imsi_prefix": str(username[0:6])}, metricHelp='Diameter Authentication related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') #Experimental Result AVP avp_experimental_result = '' avp_experimental_result += self.generate_vendor_avp(266, 40, 10415, '') #AVP Vendor ID @@ -2470,7 +2506,10 @@ def Answer_16777216_303(self, packet_vars, avps): "event": "Unknown User", "imsi_prefix": str(imsi[0:6])}, metricHelp='Diameter Authentication related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') experimental_result = self.generate_avp(298, 40, self.int_to_hex(5001, 4)) #Result Code (DIAMETER ERROR - User Unknown) experimental_result = experimental_result + self.generate_vendor_avp(266, 40, 10415, "") #Experimental Result (297) @@ -2501,7 +2540,10 @@ def Answer_16777216_303(self, packet_vars, avps): "event": "ReAuth", "imsi_prefix": str(imsi[0:6])}, metricHelp='Diameter Authentication related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') if sub_avp_612['avp_code'] == 608: self.logTool.log(service='HSS', level='debug', message="Auth mechansim requested: " + str(sub_avp_612['misc_data']), redisClient=self.redisMessaging) auth_scheme = binascii.unhexlify(sub_avp_612['misc_data']).decode('utf-8') @@ -2668,7 +2710,10 @@ def Answer_16777217_306(self, packet_vars, avps): "event": "Unknown User", "imsi_prefix": str(username[0:6])}, metricHelp='Diameter Authentication related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') result_code = 5001 #Experimental Result AVP avp_experimental_result = '' @@ -3286,7 +3331,10 @@ def Answer_16777252_324(self, packet_vars, avps): metricLabels={ "response": EquipmentStatus}, metricHelp='Diameter EIR event related Counters', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') except Exception as e: self.logTool.log(service='HSS', level='error', message=traceback.format_exc(), redisClient=self.redisMessaging) diff --git a/lib/diameterAsync.py b/lib/diameterAsync.py old mode 100644 new mode 100755 diff --git a/lib/logtool.py b/lib/logtool.py old mode 100644 new mode 100755 diff --git a/lib/lte.py b/lib/lte.py old mode 100644 new mode 100755 diff --git a/lib/messaging.py b/lib/messaging.py old mode 100644 new mode 100755 diff --git a/lib/messagingAsync.py b/lib/messagingAsync.py old mode 100644 new mode 100755 diff --git a/lib/metrics.py b/lib/metrics.py old mode 100644 new mode 100755 diff --git a/lib/milenage.py b/lib/milenage.py old mode 100644 new mode 100755 diff --git a/lib/mongodb_insert_subscriber.py b/lib/mongodb_insert_subscriber.py old mode 100644 new mode 100755 diff --git a/services/apiService.py b/services/apiService.py old mode 100644 new mode 100755 index e371e84..1648315 --- a/services/apiService.py +++ b/services/apiService.py @@ -1921,7 +1921,10 @@ def patch(self): "endpoint": "HSS", "geored_host": request.remote_addr, }, - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') if 'serving_apn' in json_data: print("Updating serving APN") if 'serving_pgw_realm' not in json_data: @@ -1947,7 +1950,10 @@ def patch(self): "endpoint": "PCRF", "geored_host": request.remote_addr, }, - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') if 'scscf' in json_data: print("Updating Serving SCSCF") if 'scscf_realm' not in json_data: @@ -1964,7 +1970,10 @@ def patch(self): "endpoint": "IMS_SCSCF", "geored_host": request.remote_addr, }, - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') if 'pcscf' in json_data: print("Updating Proxy SCSCF") if 'pcscf_realm' not in json_data: @@ -1983,7 +1992,10 @@ def patch(self): "endpoint": "IMS_PCSCF", "geored_host": request.remote_addr, }, - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') if 'imei' in json_data: print("Updating EIR") response_data.append(databaseClient.Store_IMSI_IMEI_Binding(str(json_data['imsi']), str(json_data['imei']), str(json_data['match_response_code']), propagate=False)) @@ -1994,7 +2006,10 @@ def patch(self): "endpoint": "IMEI", "geored_host": request.remote_addr, }, - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') if 'auc_id' in json_data: print("Updating AuC") response_data.append(databaseClient.Update_AuC(json_data['auc_id'], json_data['sqn'], propagate=False)) @@ -2005,7 +2020,10 @@ def patch(self): "endpoint": "SQN", "geored_host": request.remote_addr, }, - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') if 'emergency_subscriber_ip' in json_data: """ If we receive a geored payload containing emergency_subscriber_id, create or update the matching emergency_subscriber_id. @@ -2049,7 +2067,10 @@ def patch(self): "endpoint": "EMERGENCY_SUBSCRIBER", "geored_host": request.remote_addr, }, - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') return response_data, 200 except Exception as E: print("Exception when updating: " + str(E)) diff --git a/services/diameterService.py b/services/diameterService.py old mode 100644 new mode 100755 index 78b1d40..696734b --- a/services/diameterService.py +++ b/services/diameterService.py @@ -88,7 +88,7 @@ async def handleActiveDiameterPeers(self): for key in stalePeers: del self.activePeers[key] await(self.logActivePeers()) - + await(self.redisPeerMessaging.setValue(key='ActiveDiameterPeers', value=json.dumps(self.activePeers), keyExpiry=86400, usePrefix=True, prefixHostname=self.hostname, prefixServiceName='diameter')) await(asyncio.sleep(1)) @@ -101,10 +101,38 @@ async def logActivePeers(self): """ Logs the number of active connections on a rolling basis. """ - activePeers = self.activePeers - if not len(activePeers) > 0: - activePeers = '' - await(self.logTool.logAsync(service='Diameter', level='info', message=f"[Diameter] [logActivePeers] {len(self.activePeers)} Active Peers {activePeers}")) + try: + activePeers = self.activePeers + if not len(activePeers) > 0: + activePeers = '' + + await(self.logTool.logAsync(service='Diameter', level='info', message=f"[Diameter] [logActivePeers] {len(self.activePeers)} Active Peers {activePeers}")) + + if isinstance(activePeers, dict): + for peerKey, peerData in activePeers.items(): + peerHost = peerData.get('diameterHostname', None) + peerConnectionStatus = peerData.get('connectionStatus', None) + if peerHost and peerConnectionStatus: + if peerConnectionStatus.lower() == 'connected': + await(self.redisMetricMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_connected_state', + metricType='gauge', metricAction='set', + metricLabels={'host': peerHost}, + metricValue=1.0, metricHelp='Connection state of diameter peers', + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) + elif peerConnectionStatus.lower() == 'disconnected': + await(self.redisMetricMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_connected_state', + metricType='gauge', metricAction='set', + metricLabels={'host': peerHost}, + metricValue=0.0, metricHelp='Connection state of diameter peers', + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) + except Exception as e: + await(self.logTool.logAsync(service='Diameter', level='info', message=f"[Diameter] [logActivePeers] Exception: {traceback.format_exc()}")) async def logProcessedMessages(self): """ @@ -123,12 +151,18 @@ async def logProcessedMessages(self): metricValue=float(self.diameterRequests), metricLabels={'benchmark_interval': self.benchmarkingInterval}, metricHelp='Number of Diameter Requests Received', - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) await(self.redisMetricMessaging.sendMetric(serviceName='diameter', metricName='prom_diam_response_count', metricType='gauge', metricAction='inc', metricLabels={'benchmark_interval': self.benchmarkingInterval}, metricValue=float(self.diameterResponses), metricHelp='Number of Diameter Responses Sent', - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) self.diameterRequests = 0 self.diameterResponses = 0 await(asyncio.sleep(benchmarkInterval)) diff --git a/services/georedService.py b/services/georedService.py old mode 100644 new mode 100755 index 14d3117..7f84d44 --- a/services/georedService.py +++ b/services/georedService.py @@ -85,7 +85,10 @@ async def sendGeored(self, asyncSession, url: str, operation: str, body: str, tr "endpoint": "geored", "http_response_code": str(responseStatusCode), "error": ""}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) break else: asyncio.ensure_future(self.redisGeoredMessaging.sendMetric(serviceName='geored', metricName='prom_http_geored', @@ -96,7 +99,10 @@ async def sendGeored(self, asyncSession, url: str, operation: str, body: str, tr "endpoint": "geored", "http_response_code": str(responseStatusCode), "error": str(response.reason)}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) except aiohttp.ClientConnectionError as e: error_message = str(e) await(self.logTool.logAsync(service='Geored', level='warning', message=f"[Geored] [sendGeored] Operation {operation} failed on {url}, with body: ({body}) and transactionId {transactionId}. Response code: {responseStatusCode}. Error Message: {e}")) @@ -109,7 +115,10 @@ async def sendGeored(self, asyncSession, url: str, operation: str, body: str, tr "endpoint": "geored", "http_response_code": "000", "error": "No matching DNS entry found"}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) else: asyncio.ensure_future(self.redisGeoredMessaging.sendMetric(serviceName='geored', metricName='prom_http_geored', metricType='counter', metricAction='inc', @@ -119,7 +128,10 @@ async def sendGeored(self, asyncSession, url: str, operation: str, body: str, tr "endpoint": "geored", "http_response_code": "000", "error": "Connection Refused"}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) except aiohttp.ServerTimeoutError: await(self.logTool.logAsync(service='Geored', level='warning', message=f"[Geored] [sendGeored] Operation {operation} timed out on {url}, with body: ({body}) and transactionId {transactionId}. Response code: {responseStatusCode}. Error Message: {e}")) asyncio.ensure_future(self.redisGeoredMessaging.sendMetric(serviceName='geored', metricName='prom_http_geored', @@ -130,7 +142,10 @@ async def sendGeored(self, asyncSession, url: str, operation: str, body: str, tr "endpoint": "geored", "http_response_code": "000", "error": "Timeout"}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) except Exception as e: await(self.logTool.logAsync(service='Geored', level='error', message=f"[Geored] [sendGeored] Operation {operation} encountered unknown error on {url}, with body: ({body}) and transactionId {transactionId}. Response code: {responseStatusCode}. Error Message: {e}")) asyncio.ensure_future(self.redisGeoredMessaging.sendMetric(serviceName='geored', metricName='prom_http_geored', @@ -141,7 +156,10 @@ async def sendGeored(self, asyncSession, url: str, operation: str, body: str, tr "endpoint": "geored", "http_response_code": "000", "error": e}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) if self.benchmarking: await(self.logTool.logAsync(service='Geored', level='info', message=f"[Geored] [sendGeored] Time taken to send individual geored request to {url}: {round(((time.perf_counter() - startTime)*1000), 3)} ms")) @@ -190,7 +208,10 @@ async def sendWebhook(self, asyncSession, url: str, operation: str, body: str, h "endpoint": "webhook", "http_response_code": str(responseStatusCode), "error": ""}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) break else: asyncio.ensure_future(self.redisWebhookMessaging.sendMetric(serviceName='webhook', metricName='prom_http_webhook', @@ -201,7 +222,10 @@ async def sendWebhook(self, asyncSession, url: str, operation: str, body: str, h "endpoint": "webhook", "http_response_code": str(responseStatusCode), "error": str(response.reason)}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) except aiohttp.ClientConnectionError as e: error_message = str(e) await(self.logTool.logAsync(service='Geored', level='warning', message=f"[Geored] [sendWebhook] Operation {operation} failed on {url}, with body: ({body}) and transactionId {transactionId}. Response code: {responseStatusCode}. Error Message: {e}")) @@ -214,7 +238,10 @@ async def sendWebhook(self, asyncSession, url: str, operation: str, body: str, h "endpoint": "webhook", "http_response_code": "000", "error": "No matching DNS entry found"}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) else: asyncio.ensure_future(self.redisWebhookMessaging.sendMetric(serviceName='webhook', metricName='prom_http_webhook', metricType='counter', metricAction='inc', @@ -224,7 +251,10 @@ async def sendWebhook(self, asyncSession, url: str, operation: str, body: str, h "endpoint": "webhook", "http_response_code": "000", "error": "Connection Refused"}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) except aiohttp.ServerTimeoutError: await(self.logTool.logAsync(service='Geored', level='warning', message=f"[Geored] [sendWebhook] Operation {operation} timed out on {url}, with body: ({body}) and transactionId {transactionId}. Response code: {responseStatusCode}. Error Message: {e}")) asyncio.ensure_future(self.redisWebhookMessaging.sendMetric(serviceName='webhook', metricName='prom_http_webhook', @@ -235,7 +265,10 @@ async def sendWebhook(self, asyncSession, url: str, operation: str, body: str, h "endpoint": "webhook", "http_response_code": "000", "error": "Timeout"}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) except Exception as e: await(self.logTool.logAsync(service='Geored', level='error', message=f"[Geored] [sendWebhook] Operation {operation} encountered unknown error on {url}, with body: ({body}) and transactionId {transactionId}. Response code: {responseStatusCode}. Error Message: {e}")) asyncio.ensure_future(self.redisWebhookMessaging.sendMetric(serviceName='webhook', metricName='prom_http_webhook', @@ -246,7 +279,10 @@ async def sendWebhook(self, asyncSession, url: str, operation: str, body: str, h "endpoint": "webhook", "http_response_code": "000", "error": e}, - metricExpiry=60)) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric')) if self.benchmarking: await(self.logTool.logAsync(service='Geored', level='info', message=f"[Geored] [sendWebhook] Time taken to send individual webhook request to {url}: {round(((time.perf_counter() - startTime)*1000), 3)} ms")) diff --git a/services/hssService.py b/services/hssService.py old mode 100644 new mode 100755 index d3414a9..48217f7 --- a/services/hssService.py +++ b/services/hssService.py @@ -58,7 +58,7 @@ def handleQueue(self): inboundTimestamp = inboundMessage.get('inbound-received-timestamp', None) try: - diameterPeers = json.loads(self.redisMessaging.getValue("ActiveDiameterPeers")) + diameterPeers = json.loads(self.redisMessaging.getValue("ActiveDiameterPeers", usePrefix=True, prefixHostname=self.hostname, prefixServiceName='diameter')) for diameterPeer in diameterPeers: if diameterPeers[diameterPeer].get('ipAddress', '') == inboundHost and diameterPeers[diameterPeer].get('port', '') == inboundPort: @@ -67,9 +67,13 @@ def handleQueue(self): metricLabels={ "host": diameterPeers[diameterPeer]['diameterHostname']}, metricValue=float(1), metricHelp='Number of Diameter Requests Recieved per Host', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') except Exception as e: + self.logTool.log(service='HSS', level='error', message=f"[HSS] [handleQueue] Error updating prom_diam_request_count_host: {traceback.format_exc()}", redisClient=self.redisMessaging) pass try: @@ -107,7 +111,7 @@ def handleQueue(self): self.logTool.log(service='HSS', level='info', message=f"[HSS] [handleQueue] [{diameterMessageTypeInbound}] Time taken to process request: {round(((time.perf_counter() - startTime)*1000), 3)} ms", redisClient=self.redisMessaging) try: - diameterPeers = json.loads(self.redisMessaging.getValue("ActiveDiameterPeers")) + diameterPeers = json.loads(self.redisMessaging.getValue("ActiveDiameterPeers", usePrefix=True, prefixHostname=self.hostname, prefixServiceName='diameter')) for diameterPeer in diameterPeers: if diameterPeers[diameterPeer].get('ipAddress', '') == inboundHost and diameterPeers[diameterPeer].get('port', '') == inboundPort: @@ -116,9 +120,13 @@ def handleQueue(self): metricLabels={ "host": diameterPeers[diameterPeer]['diameterHostname']}, metricValue=float(1), metricHelp='Number of Diameter Responses Sent per Host', - metricExpiry=60) + metricExpiry=60, + usePrefix=True, + prefixHostname=self.hostname, + prefixServiceName='metric') except Exception as e: + self.logTool.log(service='HSS', level='error', message=f"[HSS] [handleQueue] Error updating prom_diam_response_count_host: {traceback.format_exc()}", redisClient=self.redisMessaging) pass diff --git a/services/logService.py b/services/logService.py old mode 100644 new mode 100755 diff --git a/services/metricService.py b/services/metricService.py old mode 100644 new mode 100755 index 968ddc5..7aa314a --- a/services/metricService.py +++ b/services/metricService.py @@ -96,6 +96,4 @@ def getMetrics(self): '/metrics': make_wsgi_app(registry=metricService.registry) }) - #Uncomment the statement below to run a local testing instance. - prometheusWebClient.run(host='0.0.0.0', port=9191) \ No newline at end of file From 722cc887223d83b8bda01bb545b31244306c31cf Mon Sep 17 00:00:00 2001 From: davidkneipp Date: Tue, 23 Jan 2024 13:43:51 +1000 Subject: [PATCH 8/8] Fix metrics, add prom_diam_connected_state --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16fbacc..62f5fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to PyHSS are documented in this file, beginning from [Servic The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.0.1] - Unreleased +## [1.0.1] - 2023-01-23 ### Changed