classification_aldi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e79f36d6-c655-426c-87f0-1292167c5727",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Thesis Test: aldi_none_none_none_daily"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57fd9121-1205-481e-b5c3-2a7e0daa4100",
   "metadata": {},
   "source": [
    "Discord Detector ALDI with following features:\n",
    "- No use of GMM\n",
    "- No use of selecting GMM components technique\n",
    "- No PSU consideration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6d92b9c1-1b45-4f3f-a80a-502a66f6313b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import yout ALDI version here\n",
    "from aldi_none_none_none_both import ALDI\n",
    "\n",
    "aldi_id = '01'\n",
    "aldi_name = 'aldi_none_none_none_daily'\n",
    "\n",
    "# Set experiment's parameter\n",
    "dict_param_exp = {'exp_id' : 0, 'p_value' : 0.001}\n",
    "#dict_param_exp = {'exp_id' : 1, 'p_value' : 0.005}\n",
    "# dict_param_exp = {'exp_id' : 2, 'p_value' : 0.01}\n",
    "#dict_param_exp = {'exp_id' : 3, 'p_value' : 0.05}\n",
    "#dict_param_exp = {'exp_id' : 4, 'p_value' : 0.1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ba9a0453-d765-4602-826e-8d214ed90d29",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LOAD LIBRARIES\n",
    "\n",
    "import time\n",
    "import logging\n",
    "\n",
    "from datetime import datetime\n",
    "from utils import *\n",
    "from data_import_ashrae import DataImportAshrae\n",
    "from aldi_evaluation_metrics import AldiEvaluationMetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "01620964-0e47-42bb-a5fe-c916fd21be0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# AUXILLIARY VARIABLES\n",
    "\n",
    "myDataImport = DataImportAshrae()\n",
    "myEvalMetrics = AldiEvaluationMetrics()\n",
    "\n",
    "list_site_id = list(range(0,16)) \n",
    "list_site_id = [0] # DEBUG\n",
    "list_site_name = [f'Site {i}' for i in list_site_id]\n",
    "meter_type = 0\n",
    "\n",
    "agg_method = 'majority'\n",
    "\n",
    "curr_timestamp = datetime.today().strftime('%Y%m%d-%H%M')\n",
    "\n",
    "dict_all_pred_labels = {}\n",
    "dict_all_pred_labels_pred = {}\n",
    "dict_all_true_labels_or = {}\n",
    "dict_all_true_labels_and = {}\n",
    "dict_all_true_labels_majo = {}\n",
    "dict_all_true_labels_majoplus = {}\n",
    "dict_all_roc_auc_or = {}\n",
    "dict_all_roc_auc_and = {}\n",
    "dict_all_roc_auc_majo = {}\n",
    "dict_all_roc_auc_majoplus = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ee59e96b-7861-40b0-813b-36581f68775b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10/27/2021 03:41:43 PM, aldi_none_none_none_daily, INFO: aldi_none_none_none_daily starts its experiments now\n",
      "10/27/2021 03:41:43 PM, aldi_none_none_none_daily, INFO: aldi_none_none_none_daily starts its experiments now\n"
     ]
    }
   ],
   "source": [
    "# PREPARE LOGGING TECHNIQUE\n",
    "\n",
    "# create logger\n",
    "logger = logging.getLogger(aldi_name)\n",
    "logger.setLevel(logging.DEBUG)\n",
    "\n",
    "# create console handler and set level to info\n",
    "ch = logging.StreamHandler()\n",
    "ch.setLevel(logging.DEBUG)\n",
    "\n",
    "# create file handler and set level to info\n",
    "fh = logging.FileHandler(filename=f'10_thesis_results/{aldi_id}_{aldi_name}/{aldi_id}_{aldi_name}.log')\n",
    "fh.setLevel(logging.INFO)\n",
    "\n",
    "# create formatter\n",
    "formatter = logging.Formatter('%(asctime)s, %(name)s, %(levelname)s: %(message)s',\n",
    "                              datefmt='%m/%d/%Y %I:%M:%S %p')\n",
    "\n",
    "# add formatter to ch & fh\n",
    "ch.setFormatter(formatter)\n",
    "fh.setFormatter(formatter)\n",
    "\n",
    "# add ch to logger\n",
    "logger.addHandler(ch)\n",
    "logger.addHandler(fh)\n",
    "\n",
    "logger.info(aldi_name + ' starts its experiments now')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2b5e0deb-500c-40e0-addb-dcbc4431e52a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LOAD ENERGY CONSUMPTION DATA AND META DATA\n",
    "\n",
    "df_metadata = myDataImport.get_meta_data()\n",
    "df_timestamps = myDataImport.get_timestamps()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f8a36ea1-7f20-4037-8f1e-4e577f5dc341",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10/27/2021 03:48:47 PM, aldi_none_none_none_daily, DEBUG: Discord calculation: Run Site 0\n",
      "10/27/2021 03:48:47 PM, aldi_none_none_none_daily, DEBUG: Discord calculation: Run Site 0\n",
      "10/27/2021 03:49:07 PM, aldi_none_none_none_daily, INFO: Runtime: 20.8104 sec\n",
      "10/27/2021 03:49:07 PM, aldi_none_none_none_daily, INFO: Runtime: 20.8104 sec\n",
      "10/27/2021 03:49:08 PM, aldi_none_none_none_daily, INFO: Runtime: 0.3468 min\n",
      "10/27/2021 03:49:08 PM, aldi_none_none_none_daily, INFO: Runtime: 0.3468 min\n",
      "10/27/2021 03:49:08 PM, aldi_none_none_none_daily, INFO: aldi_none_none_none_daily ends discord label caluclation\n",
      "10/27/2021 03:49:08 PM, aldi_none_none_none_daily, INFO: aldi_none_none_none_daily ends discord label caluclation\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Site 0':                  timestamp building_id  is_discord\n",
      "0      2016-01-01 00:00:00           0           1\n",
      "1      2016-01-01 01:00:00           0           1\n",
      "2      2016-01-01 02:00:00           0           1\n",
      "3      2016-01-01 03:00:00           0           1\n",
      "4      2016-01-01 04:00:00           0           1\n",
      "...                    ...         ...         ...\n",
      "922315 2016-12-31 19:00:00          48           1\n",
      "922316 2016-12-31 20:00:00          48           1\n",
      "922317 2016-12-31 21:00:00          48           1\n",
      "922318 2016-12-31 22:00:00          48           1\n",
      "922319 2016-12-31 23:00:00          48           1\n",
      "\n",
      "[922320 rows x 3 columns]}\n"
     ]
    }
   ],
   "source": [
    "# RUN DISCORD DETECTOR ALDI\n",
    "\n",
    "glb_start_time = time.time()\n",
    "for site_id in list_site_id:\n",
    "    logger.debug(f'Discord calculation: Run {list_site_name[site_id]}')\n",
    "    \n",
    "    # Select relevant energy consumption data + meta data\n",
    "    df_site_meter = myDataImport.get_meter_data([meter_type], [site_id])\n",
    "        \n",
    "    # execute aldi\n",
    "    aldi = ALDI(df_meters = df_site_meter, \n",
    "                df_metadata = df_metadata, \n",
    "                m = 24, \n",
    "                col_id = 'building_id', \n",
    "                site_id=site_id, \n",
    "                meter_id=meter_type,\n",
    "                test_type='ks',\n",
    "                use_iqr=False,\n",
    "                iqr=0.25,\n",
    "                verbose=False, \n",
    "                gpu=False,\n",
    "                hourly_processing=False,\n",
    "                aldi_name=aldi_name)\n",
    "    \n",
    "    # request predicted discord label from aldi\n",
    "    df_pred_labels = aldi.get_result_df(p_value = dict_param_exp['p_value'],\n",
    "                                       forecast_out = False)\n",
    "    \n",
    "    # request predict discord label for predictor\n",
    "    df_pred_labels_pred = aldi.get_result_df(p_value = dict_param_exp['p_value'],\n",
    "                                             forecast_out = True)\n",
    "    \n",
    "    # keep track of all sites' predicted discord labels\n",
    "    dict_all_pred_labels[list_site_name[site_id]] = df_pred_labels\n",
    "    dict_all_pred_labels_pred[list_site_name[site_id]] = df_pred_labels_pred\n",
    "\n",
    "print(dict_all_pred_labels_pred)\n",
    "# adjustment to ensure that the labels have the correct format for timekeeping\n",
    "df_all_pred_labels = pd.concat(dict_all_pred_labels, axis=1)\n",
    "df_all_pred_labels.columns = df_all_pred_labels.columns.get_level_values(1)\n",
    "df_ref_timestamps = pd.DataFrame(df_timestamps.timestamp)\n",
    "df_ref_timestamps['timestamp'] = df_ref_timestamps['timestamp'].dt.date\n",
    "df_all_pred_labels['timestamp'] = df_all_pred_labels.index.date\n",
    "df_all_pred_label_hourly = pd.merge(df_ref_timestamps, df_all_pred_labels, on='timestamp')\n",
    "    \n",
    "# calculate runtime and store the value in a log file\n",
    "runtime_sec = time.time() - glb_start_time\n",
    "logger.info(f'Runtime: {round(runtime_sec, 4)} sec')\n",
    "logger.info(f'Runtime: {round(runtime_sec / 60 , 4) } min')\n",
    "logger.info(f'{aldi_name} ends discord label caluclation')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "05c9c241-057a-4f24-b50a-4c3197fd436e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10/27/2021 03:40:04 PM, aldi_none_none_none_daily, DEBUG: Evaluation: Run Site 0\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# CALCULATE DIFFERENT METRICS\n",
    "\n",
    "# 1. SITE WISE METRICS\n",
    "for site_id in list_site_id:\n",
    "    logger.debug(f'Evaluation: Run {list_site_name[site_id]}')\n",
    "    \n",
    "    # load predicted sites' discord label (daily)\n",
    "    df_curr_pred_label_daily = dict_all_pred_labels[list_site_name[site_id]]\n",
    "    \n",
    "    # load true sites' discord label + aggregate\n",
    "    df_curr_true_labels_hourly = myDataImport.get_label_data([meter_type], [site_id])\n",
    "    \n",
    "    df_curr_true_labels_or_daily = get_daily_resolution(df_hourly_data=df_curr_true_labels_hourly, \n",
    "                                                        agg_method='logic_or')\n",
    "    df_curr_true_labels_and_daily = get_daily_resolution(df_hourly_data=df_curr_true_labels_hourly, \n",
    "                                                         agg_method='logic_and')\n",
    "    df_curr_true_labels_majo_daily = get_daily_resolution(df_hourly_data=df_curr_true_labels_hourly, \n",
    "                                                          agg_method='majority')\n",
    "    df_curr_true_labels_majoplus_daily = get_daily_resolution(df_hourly_data=df_curr_true_labels_hourly, \n",
    "                                                              agg_method='majority_plus')\n",
    "    \n",
    "    # keep track of all aggregate true labels\n",
    "    dict_all_true_labels_or[list_site_name[site_id]] = df_curr_true_labels_or_daily\n",
    "    dict_all_true_labels_and[list_site_name[site_id]] = df_curr_true_labels_and_daily\n",
    "    dict_all_true_labels_majo[list_site_name[site_id]] = df_curr_true_labels_majo_daily\n",
    "    dict_all_true_labels_majoplus[list_site_name[site_id]] = df_curr_true_labels_majoplus_daily\n",
    "    \n",
    "        \n",
    "    \n",
    "    # Calculate ROC AUC metric per site & aggregation method\n",
    "    df_roc_auc_or = myEvalMetrics.get_roc_auc(df_true=df_curr_true_labels_or_daily,\n",
    "                                              df_pred=df_curr_pred_label_daily)\n",
    "    df_roc_auc_and = myEvalMetrics.get_roc_auc(df_true=df_curr_true_labels_and_daily,\n",
    "                                               df_pred=df_curr_pred_label_daily)\n",
    "    df_roc_auc_majo = myEvalMetrics.get_roc_auc(df_true=df_curr_true_labels_majo_daily,\n",
    "                                                df_pred=df_curr_pred_label_daily)\n",
    "    df_roc_auc_majoplus = myEvalMetrics.get_roc_auc(df_true=df_curr_true_labels_majoplus_daily,\n",
    "                                                    df_pred=df_curr_pred_label_daily)\n",
    "    \n",
    "    # Keep track of all ROC AUC metric values\n",
    "    dict_all_roc_auc_or[list_site_name[site_id]] = df_roc_auc_or\n",
    "    dict_all_roc_auc_and[list_site_name[site_id]] = df_roc_auc_and\n",
    "    dict_all_roc_auc_majo[list_site_name[site_id]] = df_roc_auc_majo\n",
    "    dict_all_roc_auc_majoplus[list_site_name[site_id]] = df_roc_auc_majoplus\n",
    "    \n",
    "    # Prepare Confusion matrix/class report per site & aggregation method\n",
    "    myEvalMetrics.get_class_report(df_true = df_curr_true_labels_or_daily,\n",
    "                                   df_pred = df_curr_pred_label_daily, \n",
    "                                   aldi_impl = aldi_name, \n",
    "                                   level_name = list_site_name[site_id], \n",
    "                                   meter_type = meter_type, \n",
    "                                   path = f'10_thesis_results/{aldi_id}_{aldi_name}/or-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "    myEvalMetrics.get_class_report(df_true = df_curr_true_labels_and_daily,\n",
    "                                   df_pred = df_curr_pred_label_daily, \n",
    "                                   aldi_impl = aldi_name, \n",
    "                                   level_name = list_site_name[site_id], \n",
    "                                   meter_type = meter_type, \n",
    "                                   path = f'10_thesis_results/{aldi_id}_{aldi_name}/and-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "    myEvalMetrics.get_class_report(df_true = df_curr_true_labels_majo_daily,\n",
    "                                   df_pred = df_curr_pred_label_daily, \n",
    "                                   aldi_impl = aldi_name, \n",
    "                                   level_name = list_site_name[site_id], \n",
    "                                   meter_type = meter_type, \n",
    "                                   path = f'10_thesis_results/{aldi_id}_{aldi_name}/majo-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "    myEvalMetrics.get_class_report(df_true = df_curr_true_labels_majoplus_daily,\n",
    "                                   df_pred = df_curr_pred_label_daily, \n",
    "                                   aldi_impl = aldi_name, \n",
    "                                   level_name = list_site_name[site_id], \n",
    "                                   meter_type = meter_type, \n",
    "                                   path = f'10_thesis_results/{aldi_id}_{aldi_name}/majoplus-exp_id{dict_param_exp[\"exp_id\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0012311e-ae24-4330-89b8-9ee902b87ee5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10/27/2021 03:40:12 PM, aldi_none_none_none_daily, DEBUG: Evaluation: dataset wide metrics/results\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'              precision    recall  f1-score   support\\n\\n           0       0.00      0.00      0.00     23125\\n           1       0.40      1.00      0.57     15305\\n\\n    accuracy                           0.40     38430\\n   macro avg       0.20      0.50      0.28     38430\\nweighted avg       0.16      0.40      0.23     38430\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2. DATASET WISE METRICS\n",
    "logger.debug(f'Evaluation: dataset wide metrics/results')\n",
    "\n",
    "# store all ROC AUC metric values\n",
    "df_all_roc_auc_or = pd.DataFrame.from_dict(dict_all_roc_auc_or, orient='index', columns=['roc_auc'])\n",
    "df_all_roc_auc_or.to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/or-exp_id{dict_param_exp[\"exp_id\"]}/{curr_timestamp}-roc_auc.csv')\n",
    "\n",
    "df_all_roc_auc_and = pd.DataFrame.from_dict(dict_all_roc_auc_and, orient='index', columns=[\"roc_auc\"])\n",
    "df_all_roc_auc_and.to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/and-exp_id{dict_param_exp[\"exp_id\"]}/{curr_timestamp}-roc_auc.csv')\n",
    "\n",
    "df_all_roc_auc_majo = pd.DataFrame.from_dict(dict_all_roc_auc_majo, orient='index', columns=['roc_auc'])\n",
    "df_all_roc_auc_majo.to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/majo-exp_id{dict_param_exp[\"exp_id\"]}/{curr_timestamp}-roc_auc.csv')\n",
    "\n",
    "df_all_roc_auc_majoplus = pd.DataFrame.from_dict(dict_all_roc_auc_majoplus, orient='index', columns=['roc_auc'])\n",
    "df_all_roc_auc_majoplus.to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/majoplus-exp_id{dict_param_exp[\"exp_id\"]}/{curr_timestamp}-roc_auc.csv')\n",
    "\n",
    "\n",
    "# create a single confusion matrix for each agg method\n",
    "myEvalMetrics.get_class_report(df_true = pd.concat(dict_all_true_labels_or, axis=1), \n",
    "                               df_pred = pd.concat(dict_all_pred_labels, axis=1), \n",
    "                               aldi_impl = aldi_name, \n",
    "                               level_name = 'all', \n",
    "                               meter_type = meter_type,\n",
    "                               path = f'10_thesis_results/{aldi_id}_{aldi_name}/or-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "myEvalMetrics.get_class_report(df_true = pd.concat(dict_all_true_labels_and, axis=1), \n",
    "                               df_pred = pd.concat(dict_all_pred_labels, axis=1), \n",
    "                               aldi_impl = aldi_name, \n",
    "                               level_name = 'all', \n",
    "                               meter_type = meter_type,\n",
    "                               path = f'10_thesis_results/{aldi_id}_{aldi_name}/and-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "myEvalMetrics.get_class_report(df_true = pd.concat(dict_all_true_labels_majo, axis=1), \n",
    "                               df_pred = pd.concat(dict_all_pred_labels, axis=1), \n",
    "                               aldi_impl = aldi_name, \n",
    "                               level_name = 'all', \n",
    "                               meter_type = meter_type,\n",
    "                               path = f'10_thesis_results/{aldi_id}_{aldi_name}/majo-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "myEvalMetrics.get_class_report(df_true = pd.concat(dict_all_true_labels_majoplus, axis=1), \n",
    "                               df_pred = pd.concat(dict_all_pred_labels, axis=1), \n",
    "                               aldi_impl = aldi_name, \n",
    "                               level_name = 'all', \n",
    "                               meter_type = meter_type,\n",
    "                               path = f'10_thesis_results/{aldi_id}_{aldi_name}/majoplus-exp_id{dict_param_exp[\"exp_id\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "95b472c6-34ea-459b-85f0-2b299da3b93d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 timestamp building_id  is_discord  meter\n",
      "0      2016-01-01 00:00:00           0           1      0\n",
      "1      2016-01-01 01:00:00           0           1      0\n",
      "2      2016-01-01 02:00:00           0           1      0\n",
      "3      2016-01-01 03:00:00           0           1      0\n",
      "4      2016-01-01 04:00:00           0           1      0\n",
      "...                    ...         ...         ...    ...\n",
      "922315 2016-12-31 19:00:00          48           1      0\n",
      "922316 2016-12-31 20:00:00          48           1      0\n",
      "922317 2016-12-31 21:00:00          48           1      0\n",
      "922318 2016-12-31 22:00:00          48           1      0\n",
      "922319 2016-12-31 23:00:00          48           1      0\n",
      "\n",
      "[922320 rows x 4 columns]\n",
      "(20216100, 4)\n"
     ]
    }
   ],
   "source": [
    "# 3. BUILD EXPORTABLE FILE FOR PREDICTOR\n",
    "\n",
    "df_export_labels = pd.concat(dict_all_pred_labels_pred, axis=0)\n",
    "df_export_labels['meter'] = [0] * df_export_labels.shape[0]\n",
    "df_export_labels.index = df_export_labels.index.get_level_values(1)\n",
    "\n",
    "\n",
    "print(df_export_labels)\n",
    "\n",
    "df_left_keys = myDataImport.get_timestamps_buildings(resolution='H')\n",
    "df_exportable = pd.merge(df_left_keys, df_export_labels, how=\"left\", on=[\"timestamp\", \"building_id\", \"meter\"])\n",
    "\n",
    "# Attention: NaNs are padded with 0\n",
    "df_exportable = df_exportable.fillna(0)\n",
    "df_exportable['is_discord'] = df_exportable['is_discord'].astype('int8')\n",
    "\n",
    "print(df_exportable.shape)\n",
    "\n",
    "# Export\n",
    "df_exportable['is_discord'].to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/{curr_timestamp}-discords-exp_id{dict_param_exp[\"exp_id\"]}.csv', index=False)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87e03e39-4fca-4c24-bf10-647df370f2bc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}