2. Classification_Models.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Classification Models",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4X_F0x4f4jXS"
      },
      "source": [
        "# Classification Models\n",
        "Comparing the goodness of the following Classification models\n",
        "* Logistic Regression\n",
        "* Decision Trees\n",
        "* Random Forest\n",
        "* XGBoost\n",
        "* Extra Trees \n",
        "* SVM"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nSkjLfb_5Dm8"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hmeDRJRB5FYG"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "from sklearn.svm import LinearSVC\n",
        "\n",
        "from sklearn.metrics import classification_report, accuracy_score, balanced_accuracy_score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ps5Ocpld5cSO"
      },
      "source": [
        "## Utilities"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-U9PKU-F5eAs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8cb43205-cb35-42b0-ccfb-914410e2f30d"
      },
      "source": [
        "dropColumns = ['Year', 'Quarter', 'Month', 'DayofMonth', 'Origin', 'Dest', 'DepHour', 'ArrHour', 'FlightDate']\n",
        "\n",
        "features = ['OriginAirportID', 'DestAirportID',\n",
        "            'DepTime', 'DepDelayMinutes', 'WindSpeedKmph_Origin', \n",
        "            'WindDirDegree_Origin', 'WeatherCode_Origin',\n",
        "            'precipMM_Origin', 'Visibility_Origin', 'Pressure_Origin',\n",
        "            'Cloudcover_Origin', 'DewPointF_Origin', 'WindGustKmph_Origin',\n",
        "            'tempF_Origin', 'WindChillF_Origin', 'Humidity_Origin',\n",
        "            'WindSpeedKmph_Dest', 'WindDirDegree_Dest', 'WeatherCode_Dest',\n",
        "            'precipMM_Dest', 'Visibility_Dest', 'Pressure_Dest', 'Cloudcover_Dest',\n",
        "            'DewPointF_Dest', 'WindGustKmph_Dest', 'tempF_Dest', 'WindChillF_Dest',\n",
        "            'Humidity_Dest']\n",
        "\n",
        "target = 'ArrDel15'\n",
        "\n",
        "len(features)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "28"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "leKNv5Wa5gW-"
      },
      "source": [
        "def predict(model, X_train, X_test, y_train, y_test):\n",
        "  model.fit(X_train, y_train)\n",
        "  predictions = model.predict(X_test)\n",
        "  print(classification_report(y_test, predictions))\n",
        "  print(\"Accuracy Score: \", accuracy_score(y_test, predictions))\n",
        "  print(\"Balanced Accuracy Score: \", balanced_accuracy_score(y_test, predictions))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gSFP4cIH5Pa-"
      },
      "source": [
        "## Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g9oKlnsK5RPz"
      },
      "source": [
        "data = pd.read_csv(\"drive/MyDrive/Data/dataset.csv\")\n",
        "data = data.drop(dropColumns, axis=1)\n",
        "data = data.astype({\"ArrDel15\": np.int64})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cDHlMjFm5nwc"
      },
      "source": [
        "## Separating X and y"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsVDpCS35qNi"
      },
      "source": [
        "X = data[features]\n",
        "y = data[target]\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
        "\n",
        "scaler = StandardScaler()\n",
        "scaler.fit(X)\n",
        "X_train_scaled = scaler.transform(X_train)\n",
        "X_test_scaled = scaler.transform(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_bnU0FH56GLG"
      },
      "source": [
        "## Models"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9R5Zm7-K6IA_"
      },
      "source": [
        "### Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dYSkqaruXVQC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f9cf1e5-b668-47e0-df69-eee469a2e2e4"
      },
      "source": [
        "predict(LogisticRegression(), X_train_scaled, X_test_scaled, y_train, y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.98      0.95    292422\n",
            "           1       0.89      0.69      0.77     77866\n",
            "\n",
            "    accuracy                           0.92    370288\n",
            "   macro avg       0.91      0.83      0.86    370288\n",
            "weighted avg       0.91      0.92      0.91    370288\n",
            "\n",
            "Accuracy Score:  0.9161787581558138\n",
            "Balanced Accuracy Score:  0.831485263842979\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GzCVrMg86Z7x"
      },
      "source": [
        "### Decision Trees"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aJ0b5jObXbiX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "20b28587-bfd3-4122-ef0a-0700c2dab7ba"
      },
      "source": [
        "predict(DecisionTreeClassifier(), X_train, X_test, y_train, y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.91      0.92    292377\n",
            "           1       0.68      0.71      0.70     77911\n",
            "\n",
            "    accuracy                           0.87    370288\n",
            "   macro avg       0.80      0.81      0.81    370288\n",
            "weighted avg       0.87      0.87      0.87    370288\n",
            "\n",
            "Accuracy Score:  0.869701421596163\n",
            "Balanced Accuracy Score:  0.810183580286804\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7IfLJU_L6eSs"
      },
      "source": [
        "### Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5BNzAa3ZXdu_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3cd54455-714f-4eb0-8bd3-826f16e806fd"
      },
      "source": [
        "predict(RandomForestClassifier(), X_train, X_test, y_train, y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.98      0.95    292377\n",
            "           1       0.89      0.70      0.78     77911\n",
            "\n",
            "    accuracy                           0.92    370288\n",
            "   macro avg       0.91      0.84      0.87    370288\n",
            "weighted avg       0.92      0.92      0.91    370288\n",
            "\n",
            "Accuracy Score:  0.9182339152227456\n",
            "Balanced Accuracy Score:  0.839000226365511\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qh84NEpR6iAE"
      },
      "source": [
        "### XGBoost"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G3oknjEQXhy5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a96219b7-5149-4599-dd86-f920c472910a"
      },
      "source": [
        "predict(XGBClassifier(), pd.DataFrame(X_train, columns=features), pd.DataFrame(X_test, columns=features), y_train, y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.98      0.95    292377\n",
            "           1       0.90      0.68      0.78     77911\n",
            "\n",
            "    accuracy                           0.92    370288\n",
            "   macro avg       0.91      0.83      0.86    370288\n",
            "weighted avg       0.92      0.92      0.91    370288\n",
            "\n",
            "Accuracy Score:  0.917280603206153\n",
            "Balanced Accuracy Score:  0.8311564830055568\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0jXlrUuV6oCN"
      },
      "source": [
        "### Extra Trees"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y1t1kbCKXgEo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "12794e46-a46d-4a27-f987-ba70365a76bd"
      },
      "source": [
        "predict(ExtraTreesClassifier(), X_train_scaled, X_test_scaled, y_train, y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.97      0.94    292377\n",
            "           1       0.87      0.67      0.76     77911\n",
            "\n",
            "    accuracy                           0.91    370288\n",
            "   macro avg       0.90      0.82      0.85    370288\n",
            "weighted avg       0.91      0.91      0.91    370288\n",
            "\n",
            "Accuracy Score:  0.9103076524218986\n",
            "Balanced Accuracy Score:  0.8220946897652577\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-AKP-Rrt6p8F"
      },
      "source": [
        "### SVM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DvylSPHLXT0k",
        "outputId": "0ee7506c-adc1-44cd-dce7-85448d791875"
      },
      "source": [
        "predict(LinearSVC(), X_train_scaled, X_test_scaled, y_train, y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
            "  \"the number of iterations.\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      0.98      0.95    292377\n",
            "           1       0.90      0.68      0.77     77911\n",
            "\n",
            "    accuracy                           0.92    370288\n",
            "   macro avg       0.91      0.83      0.86    370288\n",
            "weighted avg       0.92      0.92      0.91    370288\n",
            "\n",
            "Accuracy Score:  0.9165595428423281\n",
            "Balanced Accuracy Score:  0.8301820603405525\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}