From bc74117fda8660d6327046e788be85aa56fd6a5c Mon Sep 17 00:00:00 2001
From: Petya Vasileva <petya.tsv.vasileva@gmail.com>
Date: Sun, 10 Dec 2023 16:25:34 +0200
Subject: [PATCH] Add metadata storage in ParquetUpdater

---
 src/model/Updater.py    | 10 +++++++++-
 src/pages/site.py       | 10 ++--------
 src/pages/throughput.py |  4 +++-
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/model/Updater.py b/src/model/Updater.py
index 2314fca..3571252 100644
--- a/src/model/Updater.py
+++ b/src/model/Updater.py
@@ -38,6 +38,7 @@ def __init__(self):
             self.cacheIndexData()
             self.storeAlarms()
             self.storePathChangeDescDf()
+            self.storeMetaData()
             self.storeThroughputDataAndModel()
             self.storePacketLossDataAndModel()
 
@@ -45,6 +46,7 @@ def __init__(self):
             Scheduler(60*60, self.cacheIndexData)
             Scheduler(60*10, self.storeAlarms)
             Scheduler(60*30, self.storePathChangeDescDf)
+            Scheduler(int(60*60*12), self.storeMetaData)
 
             # Store the data for the Major Alarms analysis
             Scheduler(int(60*60*12), self.storeThroughputDataAndModel)
@@ -57,7 +59,7 @@ def __init__(self):
     # taking into account the most recent 24 hours only
     def groupAlarms(self, pivotFrames):
         dateFrom, dateTo = hp.defaultTimeRange(1)
-        metaDf = qrs.getMetaData()
+        metaDf = self.pq.readFile('parquet/raw/metaDf.parquet')
 
         nodes = metaDf[~(metaDf['site'].isnull()) & ~(
             metaDf['site'] == '') & ~(metaDf['lat'] == '') & ~(metaDf['lat'].isnull())]
@@ -157,6 +159,12 @@ def cacheIndexData(self):
     #     self.pq.writeToFile(altPaths, f'{location}altPaths.parquet')
 
 
+    @timer
+    def storeMetaData(self):
+        metaDf = qrs.getMetaData()
+        self.pq.writeToFile(metaDf, "parquet/raw/metaDf.parquet")
+
+
     @timer
     def storeAlarms(self):
         dateFrom, dateTo = hp.defaultTimeRange(60)
diff --git a/src/pages/site.py b/src/pages/site.py
index 8f0fc4a..3b3ff51 100644
--- a/src/pages/site.py
+++ b/src/pages/site.py
@@ -99,13 +99,6 @@ def generate_tables(site, dateFrom, dateTo, frames, pivotFrames, alarms4Site, al
     return out
 
 
-
-@lru_cache(maxsize=None)
-def getMetaData():
-    metaDf = qrs.getMetaData()
-    return metaDf
-
-
 @lru_cache(maxsize=None)
 def loadAllTests(pq):
     measures = pq.readFile('parquet/raw/measures.parquet')
@@ -113,7 +106,8 @@ def loadAllTests(pq):
 
 
 def SitesOverviewPlots(site_name, pq):
-    metaDf = getMetaData()
+    metaDf = pq.readFile('parquet/raw/metaDf.parquet')
+
     alltests = loadAllTests(pq)
 
     units = {
diff --git a/src/pages/throughput.py b/src/pages/throughput.py
index 7b68cb2..19c6a05 100644
--- a/src/pages/throughput.py
+++ b/src/pages/throughput.py
@@ -18,6 +18,7 @@
 from utils.helpers import timer
 from model.Alarms import Alarms
 import model.queries as qrs
+from utils.parquet import Parquet
 
 
 urllib3.disable_warnings()
@@ -50,7 +51,8 @@ def convertTime(ts):
 # TODO: move that query to queries.py
 @timer
 def getRawDataFromES(src, dest, ipv6, dateFrom, dateTo):
-    metaDf = qrs.getMetaData()
+    pq = Parquet()
+    metaDf = pq.readFile('parquet/raw/metaDf.parquet')
     sips = metaDf[(metaDf['site'] == src) | (metaDf['netsite'] == src)]['ip'].values.tolist()
     sips = [ip.upper() for ip in sips] + [ip.lower() for ip in sips]
     dips = metaDf[(metaDf['site'] == dest) | (metaDf['netsite'] == dest)]['ip'].values.tolist()