Update clustering.py

TRB_label_assist/SVM_decision_boundaries.ipynb

@@ -20,7 +20,6 @@
                 "import numpy as np\n",
                 "import matplotlib.pyplot as plt\n",
                 "import matplotlib\n",
-                "import itertools\n",
                 "\n",
                 "from sklearn.pipeline import make_pipeline\n",
                 "from sklearn.preprocessing import StandardScaler\n",
@@ -67,7 +66,7 @@
                 "for u in uuids:\n",
                 "    ts = esta.TimeSeries.get_time_series(u)\n",
                 "    ct_entry[u]=eamtr._get_training_data(u,None)\n",
-                "    ct_df = ts.to_data_df(\"analysis/confirmed_trip\",itertools.chain(ct_entry[u]))\n",
+                "    ct_df = ts.to_data_df(\"analysis/confirmed_trip\",ct_entry[u])\n",
                 "    confirmed_trip_df_map[u] = ct_df\n",
                 "    labeled_trip_df_map[u] = esdtq.filter_labeled_trips(ct_df)\n",
                 "    expanded_trip_df_map[u] = esdtq.expand_userinputs(labeled_trip_df_map[u])"
@@ -270,9 +269,9 @@
     ],
     "metadata": {
         "kernelspec": {
-            "display_name": "emission-private-eval",
+            "display_name": "Python3",
             "language": "python",
-            "name": "emission-private-eval"
+            "name": "Python3"
         },
         "language_info": {
             "codemirror_mode": {
@@ -284,7 +283,7 @@
             "name": "python",
             "nbconvert_exporter": "python",
             "pygments_lexer": "ipython3",
-            "version": "3.9.16"
+            "version": "3.9.6"
         },
         "orig_nbformat": 4,
         "vscode": {

TRB_label_assist/classification_performance.ipynb

@@ -19,7 +19,6 @@
                 "import pandas as pd\n",
                 "import numpy as np\n",
                 "from uuid import UUID\n",
-                "import itertools\n",
                 "import matplotlib.pyplot as plt\n",
                 "\n",
                 "# import logging\n",
@@ -53,7 +52,7 @@
                 "for u in all_users:\n",
                 "    ts = esta.TimeSeries.get_time_series(u)\n",
                 "    ct_entry[u]=eamtr._get_training_data(u,None)\n",
-                "    ct_df = ts.to_data_df(\"analysis/confirmed_trip\",itertools.chain(ct_entry[u]))\n",
+                "    ct_df = ts.to_data_df(\"analysis/confirmed_trip\",ct_entry[u])\n",
                 "    confirmed_trip_df_map[u] = ct_df\n",
                 "    labeled_trip_df_map[u] = esdtq.filter_labeled_trips(ct_df)\n",
                 "    expanded_labeled_trip_df_map[u] = esdtq.expand_userinputs(\n",

TRB_label_assist/cluster_performance.ipynb

@@ -15,7 +15,6 @@
             "source": [
                 "%load_ext autoreload\n",
                 "%autoreload 2\n",
-                "import itertools\n",
                 "import pandas as pd\n",
                 "import matplotlib.pyplot as plt\n",
                 "from matplotlib.gridspec import GridSpec\n",
@@ -49,7 +48,7 @@
                 "for u in all_users:\n",
                 "    ts = esta.TimeSeries.get_time_series(u)\n",
                 "    ct_entry[u]=eamtr._get_training_data(u,None)    \n",
-                "    ct_df = ts.to_data_df(\"analysis/confirmed_trip\",itertools.chain(ct_entry[u]))    \n",
+                "    ct_df = ts.to_data_df(\"analysis/confirmed_trip\",ct_entry[u])    \n",
                 "    confirmed_trip_df_map[u] = ct_df\n",
                 "    labeled_trip_df_map[u] = esdtq.filter_labeled_trips(ct_df)\n",
                 "    expanded_labeled_trip_df_map[u] = esdtq.expand_userinputs(\n",

TRB_label_assist/clustering.py

@@ -131,6 +131,7 @@ def add_loc_clusters(
                 "similarity_threshold_meters": r,  # meters,
                 "apply_cutoff": False,
                 "clustering_way": clustering_way,
+                "shouldFilter":False,
                 "incremental_evaluation": False
             }    
 

TRB_label_assist/models.py

@@ -121,13 +121,13 @@ class Cluster(SetupMixin, metaclass=ABCMeta):
     """ blueprint for clustering models. """
 
     @abstractmethod
-    def fit(self, train_df,ct_entry=None):
+    def fit(self, train_df,unused=None):
         """ Fit the clustering algorithm.  
 
             Args: 
                 train_df (DataFrame): dataframe of labeled trips
-                ct_entry (List) : A list of Entry type of labeled and unlabeled trips 
-
+                unused (List) : A list of Entry type of labeled and unlabeled trips which is not used in current function. 
+                                Passed to keep fit function generic.            
             Returns:
                 self
         """
@@ -165,13 +165,13 @@ def fit_predict(self, train_df):
 class TripClassifier(SetupMixin, metaclass=ABCMeta):
 
     @abstractmethod
-    def fit(self, train_df,ct_entry=None):
+    def fit(self, train_df,unused=None):
         """ Fit a classification model.  
 
             Args: 
                 train_df (DataFrame): dataframe of labeled trips
-                ct_entry (List) : A list of Entry type of labeled and unlabeled trips 
-
+                unused (List) : A list of Entry type of labeled and unlabeled trips which is not used in current function. 
+                                Passed to keep fit function generic.            
             Returns:
                 self
         """
@@ -300,7 +300,7 @@ def set_params(self, params):
 
         return self
 
-    def fit(self, train_df,ct_entry=None):
+    def fit(self, train_df,ct_entry_list=None):
         # clean data
         logging.info("PERF: Fitting RefactoredNaiveCluster with size %s" % len(train_df))
         self.train_df = self._clean_data(train_df)
@@ -335,7 +335,7 @@ def fit(self, train_df,ct_entry=None):
 
         # fit the bins
         self.sim_model= eamtg.GreedySimilarityBinning(model_config)
-        cleaned_trip_entry= clustering.cleanEntryTypeData(self.train_df,ct_entry)
+        cleaned_trip_entry= clustering.cleanEntryTypeData(self.train_df,ct_entry_list)
         self.sim_model.fit(cleaned_trip_entry)
 
         labels = [int(l) for l in self.sim_model.tripLabels]
@@ -351,6 +351,7 @@ def predict(self, test_df):
         elif self.loc_type == 'end':
             bins = self.sim_model.bins
 
+        bins = {int(key):value for key,value in bins.items()}
         labels = []
 
         # for each trip in the test list:
@@ -380,7 +381,7 @@ def _match(self, trip, bin, loc_type):
             copied from the Similarity class on the e-mission-server. 
         """
         for t_idx in bin:
-            trip_in_bin = self.train_df.iloc[int(t_idx)]
+            trip_in_bin = self.train_df.iloc[t_idx]
             if not self._distance_helper(trip, trip_in_bin, loc_type):
                 return False
         return True
@@ -457,7 +458,7 @@ def set_params(self, params):
 
         return self
 
-    def fit(self, train_df,ct_entry=None):
+    def fit(self, train_df,unused=None):
         """ Creates clusters of trip points. 
             self.train_df will be updated with columns containing base and 
             final clusters. 
@@ -468,8 +469,8 @@ def fit(self, train_df,ct_entry=None):
 
             Args:
                 train_df (dataframe): dataframe of labeled trips
-                ct_entry (List) : A list of Entry type of labeled and unlabeled trips 
-        """
+                unused (List) : A list of Entry type of labeled and unlabeled trips which is not used in current function. 
+                                Passed to keep fit function generic.        """
         ##################
         ### clean data ###
         ##################
@@ -662,7 +663,7 @@ def set_params(self, params):
 
         return self
 
-    def fit(self, train_df,ct_entry=None):
+    def fit(self, train_df,unused=None):
         logging.info("PERF: Fitting NaiveBinningClassifier")
         # (copied from bsm.build_user_model())
 
@@ -902,13 +903,13 @@ def set_params(self, params):
 
         return self
 
-    def fit(self, train_df,ct_entry=None):
+    def fit(self, train_df,ct_entry_list=None):
         # fit clustering model
-        self.end_cluster_model.fit(train_df,ct_entry)
+        self.end_cluster_model.fit(train_df,ct_entry_list)
         self.train_df = self.end_cluster_model.train_df
 
         if self.cluster_method in ['trip', 'combination']:
-            self.start_cluster_model.fit(train_df,ct_entry)
+            self.start_cluster_model.fit(train_df,ct_entry_list)
             self.train_df.loc[:, ['start_cluster_idx'
                                   ]] = self.start_cluster_model.train_df[[
                                       'start_cluster_idx'
@@ -1071,7 +1072,7 @@ class EnsembleClassifier(TripClassifier, metaclass=ABCMeta):
     replaced_predictor = NotImplemented
 
     # required methods
-    def fit(self, train_df,ct_entry=None):
+    def fit(self, train_df,unused=None):
         # get location features
         if self.loc_feature == 'cluster':
             # fit clustering model(s) and one-hot encode their indices

TRB_label_assist/regenerate_classification_performance_results.py

@@ -1,7 +1,6 @@
 import pandas as pd
 import numpy as np
 from uuid import UUID
-import itertools
 import emission.storage.timeseries.abstract_timeseries as esta
 import emission.storage.decorations.trip_queries as esdtq
 import emission.analysis.modelling.trip_model.run_model as eamtr
@@ -20,7 +19,7 @@
 for u in all_users:
     ts = esta.TimeSeries.get_time_series(u)
     ct_entry[u]=eamtr._get_training_data(u,None)    
-    ct_df = ts.to_data_df("analysis/confirmed_trip",itertools.chain(ct_entry[u]))    
+    ct_df = ts.to_data_df("analysis/confirmed_trip",ct_entry[u])   
     confirmed_trip_df_map[u] = ct_df
     labeled_trip_df_map[u] = esdtq.filter_labeled_trips(ct_df)
     expanded_labeled_trip_df_map[u] = esdtq.expand_userinputs(