1. Data_Preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "orig_nbformat": 2,
    "kernelspec": {
      "name": "python385jvsc74a57bd0ef2904c27a51ff98685eed6f6fbf89a79855f8c8d9661ade4f9eaa9aaf909b19",
      "display_name": "Python 3.8.5 64-bit ('base': conda)"
    },
    "colab": {
      "name": "Data Preprocessing ",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VRLk1IQfQeIm"
      },
      "source": [
        "# Module 1: Data Preprocessing\n",
        "The weather folder consists of the weather data at 15\n",
        "different airports in the USA between 2016 and 2017. The flight data of all the flights that flew\n",
        "inside the USA is provided. Merge these two data appropriately such that each record of the\n",
        "flight should have the corresponding weather data available.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VgEhaNLCQeI7"
      },
      "source": [
        "## 1. Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FZs55sf2QeI-",
        "outputId": "d417c9fa-e469-4e4c-8511-5fd5d74ec710"
      },
      "source": [
        "!python -m pip install 'fsspec>=0.3.3'\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from dask import delayed\n",
        "import dask.dataframe as dd\n",
        "import os\n",
        "import json\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: fsspec>=0.3.3 in /usr/local/lib/python3.7/dist-packages (2021.5.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wbznlsjQQrOP",
        "outputId": "971aac35-6975-47d2-b2d0-7634d59939f1"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LwPYhYIyQeI_"
      },
      "source": [
        "## 2. Utilities"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ivRhZ4c_QeJA"
      },
      "source": [
        "basePath = \"drive/MyDrive/Data\"\n",
        "\n",
        "airports = [\"ATL\", \"CLT\", \"DEN\", \"DFW\", \"EWR\", \"IAH\", \"JFK\", \"LAS\", \"LAX\", \"MCO\", \"MIA\", \"ORD\", \"PHX\", \"SEA\", \"SFO\"]\n",
        "\n",
        "flightCols = [\"Origin\", \"Dest\", \"FlightDate\", \"Quarter\", \"Year\", \"Month\", \"DayofMonth\", \"DepTime\", \"DepDel15\", \"CRSDepTime\", \"DepDelayMinutes\", \"OriginAirportID\", \"DestAirportID\", \"ArrTime\", \"CRSArrTime\", \"ArrDel15\", \"ArrDelayMinutes\"]\n",
        "\n",
        "weatherCols = [\"Date\", \"Month\", \"Year\", \"Airport\", \"WindSpeedKmph\", \"WindDirDegree\", \"WeatherCode\", \"precipMM\", \"Visibility\", \"Pressure\", \"Cloudcover\", \"DewPointF\", \"WindGustKmph\", \"tempF\", \"WindChillF\", \"Humidity\", \"Time\"]\n",
        "\n",
        "weatherKeys = [\"windspeedKmph\", \"winddirDegree\", \"weatherCode\", \"precipMM\", \"visibility\", \"pressure\", \"cloudcover\", \"DewPointF\", \"WindGustKmph\", \"tempF\", \"WindChillF\", \"humidity\", \"time\"]\n",
        "\n",
        "flightColsW = [\"OriginWindSpeedKmph\", \"OriginWindDirDegree\", \"OriginWeatherCode\", \"OriginprecipMM\", \"OriginVisibility\", \"OriginPressure\", \"OriginCloudcover\", \"OriginDewPointF\", \"OriginWindGustKmph\", \"OrigintempF\", \"OriginWindChillF\", \"OriginHumidity\", \"OriginTime\", \"DestWindSpeedKmph\", \"DestWindDirDegree\", \"DestWeatherCode\", \"DestprecipMM\", \"DestVisibility\", \"DestPressure\", \"DestCloudcover\", \"DestDewPointF\", \"DestWindGustKmph\", \"DesttempF\", \"DestWindChillF\", \"DestHumidity\", \"DestTime\"]\n",
        "\n",
        "years = ['2016', '2017']\n",
        "\n",
        "months = range(1, 13)\n",
        "\n",
        "merge_right = [\"Year\", \"Month\", \"Date\", \"Airport\", \"Time\"]\n",
        "merge_left_origin = [\"Year\", \"Month\", \"DayofMonth\", \"Origin\", \"DepHour\"]\n",
        "merge_left_dest = [\"Year\", \"Month\", \"DayofMonth\", \"Dest\", \"ArrHour\"]\n",
        "\n",
        "merge_drop = [\"Date_Origin\", \"Airport_Origin\", \"Time_Origin\", \"Date_Dest\", \"Airport_Dest\", \"Time_Dest\"]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3nsg1EI2QeJC"
      },
      "source": [
        "# To Extract hour from given time\n",
        "def hour(x):\n",
        "    if len(str(x)) > 2:\n",
        "        return int(str(x)[-3::-1][::-1]+\"00\")\n",
        "    return 100\n",
        "\n",
        "# To convert given JSON to DF\n",
        "def jsonToDF(weather, month, year, airport):\n",
        "    temp = []\n",
        "    for date in weather:\n",
        "        for hour in date[\"hourly\"]:\n",
        "            row = [weather.index(date)+1, month, year, airport]\n",
        "            for key in weatherKeys:\n",
        "                row.append(hour[key])\n",
        "            temp.append(row)\n",
        "    return pd.DataFrame(temp, columns=weatherCols)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SEOwiwZ4QeJD"
      },
      "source": [
        "## 3. Reading Flight Data \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9dqVywB-QeJE",
        "outputId": "c07aa562-da77-44f5-e109-ba5151c12c4f"
      },
      "source": [
        "# Reading Flight Data (2 min)\n",
        "flightDF = dd.read_csv(os.path.join(basePath, 'flight-data', '*', '*', '*.csv'), usecols=flightCols)\n",
        "flightDF = flightDF.compute()\n",
        "\n",
        "# Retaining only required airports\n",
        "flightDF = flightDF[flightDF[\"Origin\"].isin(airports)].dropna()\n",
        "flightDF = flightDF[flightDF[\"Dest\"].isin(airports)]\n",
        "\n",
        "# Setting times to int\n",
        "flightDF = flightDF.astype({'ArrTime': np.int64, 'DepTime': np.int64})\n",
        "\n",
        "# Resetting indices\n",
        "flightDF = flightDF.reset_index(drop=True)\n",
        "\n",
        "# Calculating Time as hour alone\n",
        "flightDF[\"DepHour\"] = pd.Series([hour(x) for x in flightDF[\"DepTime\"]])\n",
        "flightDF[\"ArrHour\"] = pd.Series([hour(x) for x in flightDF[\"ArrTime\"]])\n",
        "\n",
        "\n",
        "# Saving FlightDF\n",
        "flightDF.to_csv(os.path.join(basePath, \"flight-data\", \"FlightData.csv\"), index=False) \n",
        "\n",
        "flightDF.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1851436, 19)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "id": "tPzozvX9QeJH",
        "outputId": "dc10cfb1-20f6-4b2c-ff5a-3b26f21fd004"
      },
      "source": [
        "flightDF = pd.read_csv(os.path.join(basePath, \"flight-data\", \"FlightData.csv\"))\n",
        "flightDF.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Year</th>\n",
              "      <th>Quarter</th>\n",
              "      <th>Month</th>\n",
              "      <th>DayofMonth</th>\n",
              "      <th>FlightDate</th>\n",
              "      <th>OriginAirportID</th>\n",
              "      <th>Origin</th>\n",
              "      <th>DestAirportID</th>\n",
              "      <th>Dest</th>\n",
              "      <th>CRSDepTime</th>\n",
              "      <th>DepTime</th>\n",
              "      <th>DepDelayMinutes</th>\n",
              "      <th>DepDel15</th>\n",
              "      <th>CRSArrTime</th>\n",
              "      <th>ArrTime</th>\n",
              "      <th>ArrDelayMinutes</th>\n",
              "      <th>ArrDel15</th>\n",
              "      <th>DepHour</th>\n",
              "      <th>ArrHour</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016-01-01</td>\n",
              "      <td>14747</td>\n",
              "      <td>SEA</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>745</td>\n",
              "      <td>741</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1602</td>\n",
              "      <td>1610</td>\n",
              "      <td>8.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>700</td>\n",
              "      <td>1600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2016-01-02</td>\n",
              "      <td>14747</td>\n",
              "      <td>SEA</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>745</td>\n",
              "      <td>737</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1602</td>\n",
              "      <td>1613</td>\n",
              "      <td>11.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>700</td>\n",
              "      <td>1600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>2016-01-03</td>\n",
              "      <td>14747</td>\n",
              "      <td>SEA</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>745</td>\n",
              "      <td>743</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1602</td>\n",
              "      <td>1547</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>700</td>\n",
              "      <td>1500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>2016-01-04</td>\n",
              "      <td>14747</td>\n",
              "      <td>SEA</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>745</td>\n",
              "      <td>737</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1602</td>\n",
              "      <td>1551</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>700</td>\n",
              "      <td>1500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>2016-01-05</td>\n",
              "      <td>14747</td>\n",
              "      <td>SEA</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>710</td>\n",
              "      <td>708</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1527</td>\n",
              "      <td>1524</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>700</td>\n",
              "      <td>1500</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Year  Quarter  Month  ...  ArrDel15 DepHour  ArrHour\n",
              "0  2016        1      1  ...       0.0     700     1600\n",
              "1  2016        1      1  ...       0.0     700     1600\n",
              "2  2016        1      1  ...       0.0     700     1500\n",
              "3  2016        1      1  ...       0.0     700     1500\n",
              "4  2016        1      1  ...       0.0     700     1500\n",
              "\n",
              "[5 rows x 19 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_7vFhfE_QeJI"
      },
      "source": [
        "## 4. Reading Weather Data\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h_5b72R3QeJJ",
        "outputId": "1bfaf463-b124-462d-ee36-30a1fc835fe9"
      },
      "source": [
        "# Reading Weather Data (5 min)\n",
        "weatherList = []\n",
        "for airport in airports:\n",
        "    for year in years: \n",
        "        for month in months:\n",
        "            # Path to each JSON fi\n",
        "            path = os.path.join(basePath, \"weather\", airport+\"/\"+str(year)+\"-\"+str(month)+\".json\")\n",
        "            with open(path) as json_file:\n",
        "                weather = json.load(json_file)[\"data\"][\"weather\"]\n",
        "            # Append each DF into weatherList \n",
        "            weatherList.append(delayed(jsonToDF)(weather, month, year, airport).compute())\n",
        "\n",
        "# Merging DFs into weatherDF\n",
        "weatherDF = pd.concat(weatherList, ignore_index=True)\n",
        "weatherDF[\"Time\"] = [int(x) + 100 for x in weatherDF[\"Time\"]]\n",
        "weatherDF.to_csv(os.path.join(basePath, \"weather\", \"WeatherData.csv\"), index=False) # Saving data into weatherData.csv\n",
        "weatherDF.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(263160, 17)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "id": "wZbw2zrXQeJK",
        "outputId": "888ec8ff-218a-40b2-d68e-68715b922ddf"
      },
      "source": [
        "weatherDF = pd.read_csv(os.path.join(basePath, \"weather\", \"WeatherData.csv\"))\n",
        "weatherDF.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Date</th>\n",
              "      <th>Month</th>\n",
              "      <th>Year</th>\n",
              "      <th>Airport</th>\n",
              "      <th>WindSpeedKmph</th>\n",
              "      <th>WindDirDegree</th>\n",
              "      <th>WeatherCode</th>\n",
              "      <th>precipMM</th>\n",
              "      <th>Visibility</th>\n",
              "      <th>Pressure</th>\n",
              "      <th>Cloudcover</th>\n",
              "      <th>DewPointF</th>\n",
              "      <th>WindGustKmph</th>\n",
              "      <th>tempF</th>\n",
              "      <th>WindChillF</th>\n",
              "      <th>Humidity</th>\n",
              "      <th>Time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016</td>\n",
              "      <td>ATL</td>\n",
              "      <td>11</td>\n",
              "      <td>318</td>\n",
              "      <td>176</td>\n",
              "      <td>0.1</td>\n",
              "      <td>10</td>\n",
              "      <td>1023</td>\n",
              "      <td>100</td>\n",
              "      <td>47</td>\n",
              "      <td>17</td>\n",
              "      <td>49</td>\n",
              "      <td>46</td>\n",
              "      <td>91</td>\n",
              "      <td>100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016</td>\n",
              "      <td>ATL</td>\n",
              "      <td>13</td>\n",
              "      <td>317</td>\n",
              "      <td>176</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1023</td>\n",
              "      <td>100</td>\n",
              "      <td>44</td>\n",
              "      <td>22</td>\n",
              "      <td>46</td>\n",
              "      <td>42</td>\n",
              "      <td>92</td>\n",
              "      <td>200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016</td>\n",
              "      <td>ATL</td>\n",
              "      <td>14</td>\n",
              "      <td>315</td>\n",
              "      <td>122</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1023</td>\n",
              "      <td>100</td>\n",
              "      <td>41</td>\n",
              "      <td>26</td>\n",
              "      <td>43</td>\n",
              "      <td>38</td>\n",
              "      <td>92</td>\n",
              "      <td>300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016</td>\n",
              "      <td>ATL</td>\n",
              "      <td>16</td>\n",
              "      <td>314</td>\n",
              "      <td>122</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1023</td>\n",
              "      <td>100</td>\n",
              "      <td>38</td>\n",
              "      <td>30</td>\n",
              "      <td>40</td>\n",
              "      <td>33</td>\n",
              "      <td>93</td>\n",
              "      <td>400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016</td>\n",
              "      <td>ATL</td>\n",
              "      <td>17</td>\n",
              "      <td>314</td>\n",
              "      <td>122</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1023</td>\n",
              "      <td>100</td>\n",
              "      <td>38</td>\n",
              "      <td>30</td>\n",
              "      <td>40</td>\n",
              "      <td>33</td>\n",
              "      <td>93</td>\n",
              "      <td>500</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Date  Month  Year Airport  ...  tempF  WindChillF  Humidity  Time\n",
              "0     1      1  2016     ATL  ...     49          46        91   100\n",
              "1     1      1  2016     ATL  ...     46          42        92   200\n",
              "2     1      1  2016     ATL  ...     43          38        92   300\n",
              "3     1      1  2016     ATL  ...     40          33        93   400\n",
              "4     1      1  2016     ATL  ...     40          33        93   500\n",
              "\n",
              "[5 rows x 17 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zPGdaOf6QeJM"
      },
      "source": [
        "## 5. Merging Flight and Weather Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gHksMnrMQeJM",
        "outputId": "5ae7b1e7-5fcf-481d-903d-f85568d3f7b8"
      },
      "source": [
        "# Adding flight origin weather details\n",
        "data = flightDF.merge(weatherDF, left_on=merge_left_origin, right_on=merge_right)\n",
        "\n",
        "# Adding flight destination weather details\n",
        "data = data.merge(weatherDF, left_on=merge_left_dest, right_on=merge_right, suffixes=(\"_Origin\", \"_Dest\"))\n",
        "\n",
        "# Dropping redundant columns\n",
        "data = data.drop(merge_drop, axis=1)\n",
        "\n",
        "# Saving into Data.csv\n",
        "data.to_csv(os.path.join(basePath, \"dataset.csv\"), index=False)\n",
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1851436, 43)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hgN9FVRqQeJN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "outputId": "22ca7801-baa0-4bdc-a03c-794194dcdd01"
      },
      "source": [
        "data = pd.read_csv(os.path.join(basePath, \"dataset.csv\"))\n",
        "data.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Year</th>\n",
              "      <th>Quarter</th>\n",
              "      <th>Month</th>\n",
              "      <th>DayofMonth</th>\n",
              "      <th>FlightDate</th>\n",
              "      <th>OriginAirportID</th>\n",
              "      <th>Origin</th>\n",
              "      <th>DestAirportID</th>\n",
              "      <th>Dest</th>\n",
              "      <th>CRSDepTime</th>\n",
              "      <th>DepTime</th>\n",
              "      <th>DepDelayMinutes</th>\n",
              "      <th>DepDel15</th>\n",
              "      <th>CRSArrTime</th>\n",
              "      <th>ArrTime</th>\n",
              "      <th>ArrDelayMinutes</th>\n",
              "      <th>ArrDel15</th>\n",
              "      <th>DepHour</th>\n",
              "      <th>ArrHour</th>\n",
              "      <th>WindSpeedKmph_Origin</th>\n",
              "      <th>WindDirDegree_Origin</th>\n",
              "      <th>WeatherCode_Origin</th>\n",
              "      <th>precipMM_Origin</th>\n",
              "      <th>Visibility_Origin</th>\n",
              "      <th>Pressure_Origin</th>\n",
              "      <th>Cloudcover_Origin</th>\n",
              "      <th>DewPointF_Origin</th>\n",
              "      <th>WindGustKmph_Origin</th>\n",
              "      <th>tempF_Origin</th>\n",
              "      <th>WindChillF_Origin</th>\n",
              "      <th>Humidity_Origin</th>\n",
              "      <th>WindSpeedKmph_Dest</th>\n",
              "      <th>WindDirDegree_Dest</th>\n",
              "      <th>WeatherCode_Dest</th>\n",
              "      <th>precipMM_Dest</th>\n",
              "      <th>Visibility_Dest</th>\n",
              "      <th>Pressure_Dest</th>\n",
              "      <th>Cloudcover_Dest</th>\n",
              "      <th>DewPointF_Dest</th>\n",
              "      <th>WindGustKmph_Dest</th>\n",
              "      <th>tempF_Dest</th>\n",
              "      <th>WindChillF_Dest</th>\n",
              "      <th>Humidity_Dest</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016-01-01</td>\n",
              "      <td>14747</td>\n",
              "      <td>SEA</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>745</td>\n",
              "      <td>741</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1602</td>\n",
              "      <td>1610</td>\n",
              "      <td>8.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>700</td>\n",
              "      <td>1600</td>\n",
              "      <td>7</td>\n",
              "      <td>106</td>\n",
              "      <td>113</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1030</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>8</td>\n",
              "      <td>34</td>\n",
              "      <td>29</td>\n",
              "      <td>67</td>\n",
              "      <td>21</td>\n",
              "      <td>254</td>\n",
              "      <td>113</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1016</td>\n",
              "      <td>0</td>\n",
              "      <td>33</td>\n",
              "      <td>24</td>\n",
              "      <td>42</td>\n",
              "      <td>33</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016-01-01</td>\n",
              "      <td>13204</td>\n",
              "      <td>MCO</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>1330</td>\n",
              "      <td>1408</td>\n",
              "      <td>38.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1603</td>\n",
              "      <td>1628</td>\n",
              "      <td>25.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1400</td>\n",
              "      <td>1600</td>\n",
              "      <td>8</td>\n",
              "      <td>227</td>\n",
              "      <td>122</td>\n",
              "      <td>0.1</td>\n",
              "      <td>8</td>\n",
              "      <td>1019</td>\n",
              "      <td>79</td>\n",
              "      <td>72</td>\n",
              "      <td>10</td>\n",
              "      <td>79</td>\n",
              "      <td>78</td>\n",
              "      <td>83</td>\n",
              "      <td>21</td>\n",
              "      <td>254</td>\n",
              "      <td>113</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1016</td>\n",
              "      <td>0</td>\n",
              "      <td>33</td>\n",
              "      <td>24</td>\n",
              "      <td>42</td>\n",
              "      <td>33</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016-01-01</td>\n",
              "      <td>13204</td>\n",
              "      <td>MCO</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>1445</td>\n",
              "      <td>1445</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1720</td>\n",
              "      <td>1659</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1400</td>\n",
              "      <td>1600</td>\n",
              "      <td>8</td>\n",
              "      <td>227</td>\n",
              "      <td>122</td>\n",
              "      <td>0.1</td>\n",
              "      <td>8</td>\n",
              "      <td>1019</td>\n",
              "      <td>79</td>\n",
              "      <td>72</td>\n",
              "      <td>10</td>\n",
              "      <td>79</td>\n",
              "      <td>78</td>\n",
              "      <td>83</td>\n",
              "      <td>21</td>\n",
              "      <td>254</td>\n",
              "      <td>113</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1016</td>\n",
              "      <td>0</td>\n",
              "      <td>33</td>\n",
              "      <td>24</td>\n",
              "      <td>42</td>\n",
              "      <td>33</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016-01-01</td>\n",
              "      <td>12892</td>\n",
              "      <td>LAX</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>855</td>\n",
              "      <td>853</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1712</td>\n",
              "      <td>1650</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>800</td>\n",
              "      <td>1600</td>\n",
              "      <td>12</td>\n",
              "      <td>29</td>\n",
              "      <td>113</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1020</td>\n",
              "      <td>1</td>\n",
              "      <td>30</td>\n",
              "      <td>15</td>\n",
              "      <td>50</td>\n",
              "      <td>47</td>\n",
              "      <td>47</td>\n",
              "      <td>21</td>\n",
              "      <td>254</td>\n",
              "      <td>113</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1016</td>\n",
              "      <td>0</td>\n",
              "      <td>33</td>\n",
              "      <td>24</td>\n",
              "      <td>42</td>\n",
              "      <td>33</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2016-01-01</td>\n",
              "      <td>12892</td>\n",
              "      <td>LAX</td>\n",
              "      <td>12478</td>\n",
              "      <td>JFK</td>\n",
              "      <td>840</td>\n",
              "      <td>842</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1708</td>\n",
              "      <td>1626</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>800</td>\n",
              "      <td>1600</td>\n",
              "      <td>12</td>\n",
              "      <td>29</td>\n",
              "      <td>113</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1020</td>\n",
              "      <td>1</td>\n",
              "      <td>30</td>\n",
              "      <td>15</td>\n",
              "      <td>50</td>\n",
              "      <td>47</td>\n",
              "      <td>47</td>\n",
              "      <td>21</td>\n",
              "      <td>254</td>\n",
              "      <td>113</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "      <td>1016</td>\n",
              "      <td>0</td>\n",
              "      <td>33</td>\n",
              "      <td>24</td>\n",
              "      <td>42</td>\n",
              "      <td>33</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Year  Quarter  Month  ...  tempF_Dest WindChillF_Dest  Humidity_Dest\n",
              "0  2016        1      1  ...          42              33             74\n",
              "1  2016        1      1  ...          42              33             74\n",
              "2  2016        1      1  ...          42              33             74\n",
              "3  2016        1      1  ...          42              33             74\n",
              "4  2016        1      1  ...          42              33             74\n",
              "\n",
              "[5 rows x 43 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    }
  ]
}