Merge pull request #11 from maxymnaumchyk/master

src/ml/thrpt_dataset_model_train.py

@@ -65,6 +65,11 @@ def trainMLmodel(rawDf):
     # disp = disp.plot(cmap=plt.cm.YlGnBu,values_format='g')
     # plt.show()
 
+    return rawDf_onehot, model
+
+def predictData(rawDf_onehot, model):
+    rawDf_custom_x = rawDf_onehot.drop(['alarm_created'], axis=1)
+
     #preparing final datasets for further analysis
     y = model.predict(rawDf_custom_x)
     df_to_plot = rawDf_custom_x.copy()
@@ -83,5 +88,3 @@ def trainMLmodel(rawDf):
 
 
 
-
-

src/model/Updater.py

@@ -10,9 +10,13 @@
 from utils.helpers import timer
 import model.queries as qrs
 import pandas as pd
+import pickle
 
 from ml.create_thrpt_dataset import createThrptDataset
+from ml.thrpt_dataset_model_train import trainMLmodel
 from ml.create_packet_loss_dataset import createPcktDataset
+from ml.packet_loss_one_month_onehot import one_month_data
+from ml.packet_loss_train_model import packet_loss_train_model
 import os
 from datetime import datetime, timedelta
 
@@ -34,17 +38,17 @@ def __init__(self):
             self.cacheIndexData()
             self.storeAlarms()
             self.storePathChangeDescDf()
-            self.storeThroughputData()
-            self.storePacketLossData()
+            self.storeThroughputDataAndModel()
+            self.storePacketLossDataAndModel()
 
         try:
             Scheduler(3600, self.cacheIndexData)
             Scheduler(1800, self.storeAlarms)
             Scheduler(1800, self.storePathChangeDescDf)
 
             # Store the data for the Major Alarms analysis
-            Scheduler(int(60*60*12), self.storeThroughputData)
-            Scheduler(int(60*60*12), self.storePacketLossData)
+            Scheduler(int(60*60*12), self.storeThroughputDataAndModel)
+            Scheduler(int(60*60*12), self.storePacketLossDataAndModel)
         except Exception as e:
             print(traceback.format_exc())
 
@@ -194,27 +198,47 @@ def createLocation(location):
             os.mkdir(location)
 
     @timer
-    def storeThroughputData(self):
+    def storeThroughputDataAndModel(self):
         now = hp.defaultTimeRange(days=60, datesOnly=True)
         start_date = now[0]
         end_date = now[1]
         start_date, end_date = [f'{start_date}T00:01:00.000Z', f'{end_date}T23:59:59.000Z']
 
         rawDf = createThrptDataset(start_date, end_date)
 
-        self.pq.writeToFile(rawDf, f'{self.location}ml-datasets/rawDf.parquet')
+        self.pq.writeToFile(rawDf, f'{self.location}ml-datasets/throughput_Df.parquet')
+
+        # train the ML model on the loaded dataset
+        rawDf_onehot, model = trainMLmodel(rawDf)
+        del rawDf
+
+        self.pq.writeToFile(rawDf_onehot, f'{self.location}ml-datasets/throughput_onehot_Df.parquet')
+        # save the classification model as a pickle file
+        model_pkl_file = f'{self.location}ml-datasets/XGB_Classifier_model_throughput.pkl'
+        with open(model_pkl_file, 'wb') as file:
+            pickle.dump(model, file)
+
 
     @timer
-    def storePacketLossData(self):
+    def storePacketLossDataAndModel(self):
         now = hp.defaultTimeRange(days=60, datesOnly=True)
         start_date = now[0]
         end_date = now[1]
         start_date, end_date = [f'{start_date}T00:01:00.000Z', f'{end_date}T23:59:59.000Z']
 
         plsDf = createPcktDataset(start_date, end_date)
-
-        self.pq.writeToFile(plsDf, f'{self.location}ml-datasets/plsDf.parquet')
-
+        self.pq.writeToFile(plsDf, f'{self.location}ml-datasets/packet_loss_Df.parquet')
+
+        # onehot encode the whole dataset and leave only one month for further ML training
+        plsDf_onehot_month = one_month_data(plsDf)
+        # train the model on one month data
+        model = packet_loss_train_model(plsDf_onehot_month)
+        del plsDf_onehot_month
+
+        # save the classification model as a pickle file
+        model_pkl_file = f'{self.location}ml-datasets/XGB_Classifier_model_packet_loss.pkl'
+        with open(model_pkl_file, 'wb') as file:
+            pickle.dump(model, file)