Add metadata storage in ParquetUpdater

src/model/Updater.py

@@ -38,13 +38,15 @@ def __init__(self):
             self.cacheIndexData()
             self.storeAlarms()
             self.storePathChangeDescDf()
+            self.storeMetaData()
             self.storeThroughputDataAndModel()
             self.storePacketLossDataAndModel()
 
         try:
             Scheduler(60*60, self.cacheIndexData)
             Scheduler(60*10, self.storeAlarms)
             Scheduler(60*30, self.storePathChangeDescDf)
+            Scheduler(int(60*60*12), self.storeMetaData)
 
             # Store the data for the Major Alarms analysis
             Scheduler(int(60*60*12), self.storeThroughputDataAndModel)
@@ -57,7 +59,7 @@ def __init__(self):
     # taking into account the most recent 24 hours only
     def groupAlarms(self, pivotFrames):
         dateFrom, dateTo = hp.defaultTimeRange(1)
-        metaDf = qrs.getMetaData()
+        metaDf = self.pq.readFile('parquet/raw/metaDf.parquet')
 
         nodes = metaDf[~(metaDf['site'].isnull()) & ~(
             metaDf['site'] == '') & ~(metaDf['lat'] == '') & ~(metaDf['lat'].isnull())]
@@ -157,6 +159,12 @@ def cacheIndexData(self):
     #     self.pq.writeToFile(altPaths, f'{location}altPaths.parquet')
 
 
+    @timer
+    def storeMetaData(self):
+        metaDf = qrs.getMetaData()
+        self.pq.writeToFile(metaDf, "parquet/raw/metaDf.parquet")
+
+
     @timer
     def storeAlarms(self):
         dateFrom, dateTo = hp.defaultTimeRange(60)

src/pages/site.py

@@ -99,21 +99,15 @@ def generate_tables(site, dateFrom, dateTo, frames, pivotFrames, alarms4Site, al
     return out
 
 
-
-@lru_cache(maxsize=None)
-def getMetaData():
-    metaDf = qrs.getMetaData()
-    return metaDf
-
-
 @lru_cache(maxsize=None)
 def loadAllTests(pq):
     measures = pq.readFile('parquet/raw/measures.parquet')
     return measures
 
 
 def SitesOverviewPlots(site_name, pq):
-    metaDf = getMetaData()
+    metaDf = pq.readFile('parquet/raw/metaDf.parquet')
+
     alltests = loadAllTests(pq)
 
     units = {

src/pages/throughput.py

@@ -18,6 +18,7 @@
 from utils.helpers import timer
 from model.Alarms import Alarms
 import model.queries as qrs
+from utils.parquet import Parquet
 
 
 urllib3.disable_warnings()
@@ -50,7 +51,8 @@ def convertTime(ts):
 # TODO: move that query to queries.py
 @timer
 def getRawDataFromES(src, dest, ipv6, dateFrom, dateTo):
-    metaDf = qrs.getMetaData()
+    pq = Parquet()
+    metaDf = pq.readFile('parquet/raw/metaDf.parquet')
     sips = metaDf[(metaDf['site'] == src) | (metaDf['netsite'] == src)]['ip'].values.tolist()
     sips = [ip.upper() for ip in sips] + [ip.lower() for ip in sips]
     dips = metaDf[(metaDf['site'] == dest) | (metaDf['netsite'] == dest)]['ip'].values.tolist()