Titanic dataset munging.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib as ml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 891 entries, 1 to 891\n",
      "Data columns (total 11 columns):\n",
      "Survived    891 non-null int64\n",
      "Pclass      891 non-null int64\n",
      "Name        891 non-null object\n",
      "Sex         891 non-null object\n",
      "Age         714 non-null float64\n",
      "SibSp       891 non-null int64\n",
      "Parch       891 non-null int64\n",
      "Ticket      891 non-null object\n",
      "Fare        891 non-null float64\n",
      "Cabin       204 non-null object\n",
      "Embarked    889 non-null object\n",
      "dtypes: float64(2), int64(4), object(5)\n",
      "memory usage: 83.5+ KB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 418 entries, 892 to 1309\n",
      "Data columns (total 10 columns):\n",
      "Pclass      418 non-null int64\n",
      "Name        418 non-null object\n",
      "Sex         418 non-null object\n",
      "Age         332 non-null float64\n",
      "SibSp       418 non-null int64\n",
      "Parch       418 non-null int64\n",
      "Ticket      418 non-null object\n",
      "Fare        417 non-null float64\n",
      "Cabin       91 non-null object\n",
      "Embarked    418 non-null object\n",
      "dtypes: float64(2), int64(3), object(5)\n",
      "memory usage: 35.9+ KB\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('train.csv', index_col = 'PassengerId')\n",
    "test_df = pd.read_csv('test.csv', index_col = 'PassengerId')\n",
    "#train_df.info()\n",
    "#test_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "test_df['Survived'] = -888\n",
    "df = pd.concat((train_df, test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1309 entries, 1 to 1309\n",
      "Data columns (total 11 columns):\n",
      "Age         1046 non-null float64\n",
      "Cabin       295 non-null object\n",
      "Embarked    1307 non-null object\n",
      "Fare        1308 non-null float64\n",
      "Name        1309 non-null object\n",
      "Parch       1309 non-null int64\n",
      "Pclass      1309 non-null int64\n",
      "Sex         1309 non-null object\n",
      "SibSp       1309 non-null int64\n",
      "Survived    1309 non-null int64\n",
      "Ticket      1309 non-null object\n",
      "dtypes: float64(2), int64(4), object(5)\n",
      "memory usage: 162.7+ KB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Name</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Ticket</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PassengerId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>54.0</td>\n",
       "      <td>E46</td>\n",
       "      <td>S</td>\n",
       "      <td>51.8625</td>\n",
       "      <td>McCarthy, Mr. Timothy J</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>28.0</td>\n",
       "      <td>A6</td>\n",
       "      <td>S</td>\n",
       "      <td>35.5000</td>\n",
       "      <td>Sloper, Mr. William Thompson</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>113788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>19.0</td>\n",
       "      <td>C23 C25 C27</td>\n",
       "      <td>S</td>\n",
       "      <td>263.0000</td>\n",
       "      <td>Fortune, Mr. Charles Alexander</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>19950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>40.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>27.7208</td>\n",
       "      <td>Uruchurtu, Don. Manuel E</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>28.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>82.1708</td>\n",
       "      <td>Meyer, Mr. Edgar Joseph</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>42.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>52.0000</td>\n",
       "      <td>Holverson, Mr. Alexander Oskar</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>65.0</td>\n",
       "      <td>B30</td>\n",
       "      <td>C</td>\n",
       "      <td>61.9792</td>\n",
       "      <td>Ostby, Mr. Engelhart Cornelius</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>113509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>NaN</td>\n",
       "      <td>C52</td>\n",
       "      <td>S</td>\n",
       "      <td>35.5000</td>\n",
       "      <td>Woolner, Mr. Hugh</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>19947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>45.0</td>\n",
       "      <td>C83</td>\n",
       "      <td>S</td>\n",
       "      <td>83.4750</td>\n",
       "      <td>Harris, Mr. Henry Birkhardt</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>36973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>27.7208</td>\n",
       "      <td>Stewart, Mr. Albert A</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>28.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>47.1000</td>\n",
       "      <td>Carrau, Mr. Francisco M</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>113059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>46.0</td>\n",
       "      <td>E31</td>\n",
       "      <td>S</td>\n",
       "      <td>61.1750</td>\n",
       "      <td>Chaffee, Mr. Herbert Fuller</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>W.E.P. 5734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>71.0</td>\n",
       "      <td>A5</td>\n",
       "      <td>C</td>\n",
       "      <td>34.6542</td>\n",
       "      <td>Goldschmidt, Mr. George B</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>23.0</td>\n",
       "      <td>D10 D12</td>\n",
       "      <td>C</td>\n",
       "      <td>63.3583</td>\n",
       "      <td>Greenfield, Mr. William Bertram</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>PC 17759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>21.0</td>\n",
       "      <td>D26</td>\n",
       "      <td>S</td>\n",
       "      <td>77.2875</td>\n",
       "      <td>White, Mr. Richard Frasar</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>47.0</td>\n",
       "      <td>C110</td>\n",
       "      <td>S</td>\n",
       "      <td>52.0000</td>\n",
       "      <td>Porter, Mr. Walter Chamberlain</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>24.0</td>\n",
       "      <td>B58 B60</td>\n",
       "      <td>C</td>\n",
       "      <td>247.5208</td>\n",
       "      <td>Baxter, Mr. Quigg Edmond</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>54.0</td>\n",
       "      <td>D26</td>\n",
       "      <td>S</td>\n",
       "      <td>77.2875</td>\n",
       "      <td>White, Mr. Percival Wayland</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>37.0</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>Futrelle, Mr. Jacques Heath</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>24.0</td>\n",
       "      <td>B86</td>\n",
       "      <td>C</td>\n",
       "      <td>79.2000</td>\n",
       "      <td>Giglio, Mr. Victor</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>51.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>61.3792</td>\n",
       "      <td>Williams, Mr. Charles Duane</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>25.9250</td>\n",
       "      <td>Baumann, Mr. John D</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>61.0</td>\n",
       "      <td>B19</td>\n",
       "      <td>S</td>\n",
       "      <td>33.5000</td>\n",
       "      <td>Van der hoef, Mr. Wyckoff</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>111240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>56.0</td>\n",
       "      <td>A7</td>\n",
       "      <td>C</td>\n",
       "      <td>30.6958</td>\n",
       "      <td>Smith, Mr. James Clinch</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>NaN</td>\n",
       "      <td>A32</td>\n",
       "      <td>S</td>\n",
       "      <td>50.0000</td>\n",
       "      <td>Rood, Mr. Hugh Roscoe</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>113767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>45.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>26.5500</td>\n",
       "      <td>Romaine, Mr. Charles Hallace (\"Mr C Rolmane\")</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>111428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>40.0</td>\n",
       "      <td>A31</td>\n",
       "      <td>C</td>\n",
       "      <td>31.0000</td>\n",
       "      <td>Blank, Mr. Henry</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>112277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>38.0</td>\n",
       "      <td>C93</td>\n",
       "      <td>S</td>\n",
       "      <td>90.0000</td>\n",
       "      <td>Hoyt, Mr. Frederick Maxfield</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>19943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>44.0</td>\n",
       "      <td>C78</td>\n",
       "      <td>Q</td>\n",
       "      <td>90.0000</td>\n",
       "      <td>Minahan, Dr. William Edward</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>19928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>37.0</td>\n",
       "      <td>D35</td>\n",
       "      <td>S</td>\n",
       "      <td>52.5542</td>\n",
       "      <td>Beckwith, Mr. Richard Leonard</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1088</th>\n",
       "      <td>6.0</td>\n",
       "      <td>E34</td>\n",
       "      <td>C</td>\n",
       "      <td>134.5000</td>\n",
       "      <td>Spedden, Master. Robert Douglas</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>16966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1094</th>\n",
       "      <td>47.0</td>\n",
       "      <td>C62 C64</td>\n",
       "      <td>C</td>\n",
       "      <td>227.5250</td>\n",
       "      <td>Astor, Col. John Jacob</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>PC 17757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1097</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>25.7417</td>\n",
       "      <td>Omont, Mr. Alfred Fernand</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>F.C. 12998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1107</th>\n",
       "      <td>42.0</td>\n",
       "      <td>B11</td>\n",
       "      <td>S</td>\n",
       "      <td>42.5000</td>\n",
       "      <td>Head, Mr. Christopher</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>113038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1109</th>\n",
       "      <td>57.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>164.8667</td>\n",
       "      <td>Wick, Mr. George Dennick</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>36928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1126</th>\n",
       "      <td>39.0</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>Cumings, Mr. John Bradley</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>PC 17599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128</th>\n",
       "      <td>64.0</td>\n",
       "      <td>D37</td>\n",
       "      <td>C</td>\n",
       "      <td>75.2500</td>\n",
       "      <td>Warren, Mr. Frank Manley</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>110813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1134</th>\n",
       "      <td>45.0</td>\n",
       "      <td>E34</td>\n",
       "      <td>C</td>\n",
       "      <td>134.5000</td>\n",
       "      <td>Spedden, Mr. Frederic Oakley</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>16966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1137</th>\n",
       "      <td>41.0</td>\n",
       "      <td>D21</td>\n",
       "      <td>S</td>\n",
       "      <td>51.8625</td>\n",
       "      <td>Kenyon, Mr. Frederick R</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>17464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1144</th>\n",
       "      <td>27.0</td>\n",
       "      <td>C89</td>\n",
       "      <td>C</td>\n",
       "      <td>136.7792</td>\n",
       "      <td>Clark, Mr. Walter Miller</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>13508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1158</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Chisholm, Mr. Roderick Robert Crispin</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>112051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1162</th>\n",
       "      <td>46.0</td>\n",
       "      <td>C6</td>\n",
       "      <td>C</td>\n",
       "      <td>75.2417</td>\n",
       "      <td>McCaffry, Mr. Thomas Francis</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>13050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1179</th>\n",
       "      <td>24.0</td>\n",
       "      <td>B45</td>\n",
       "      <td>S</td>\n",
       "      <td>82.2667</td>\n",
       "      <td>Snyder, Mr. John Pillsbury</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>21228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1182</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>39.6000</td>\n",
       "      <td>Rheims, Mr. George Alexander Lucien</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>PC 17607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1185</th>\n",
       "      <td>53.0</td>\n",
       "      <td>A34</td>\n",
       "      <td>S</td>\n",
       "      <td>81.8583</td>\n",
       "      <td>Dodge, Dr. Washington</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>33638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1190</th>\n",
       "      <td>30.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>45.5000</td>\n",
       "      <td>Loring, Mr. Joseph Holland</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>113801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1198</th>\n",
       "      <td>30.0</td>\n",
       "      <td>C22 C26</td>\n",
       "      <td>S</td>\n",
       "      <td>151.5500</td>\n",
       "      <td>Allison, Mr. Hudson Joshua Creighton</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>113781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1200</th>\n",
       "      <td>55.0</td>\n",
       "      <td>B69</td>\n",
       "      <td>S</td>\n",
       "      <td>93.5000</td>\n",
       "      <td>Hays, Mr. Charles Melville</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>12749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1208</th>\n",
       "      <td>57.0</td>\n",
       "      <td>B78</td>\n",
       "      <td>C</td>\n",
       "      <td>146.5208</td>\n",
       "      <td>Spencer, Mr. William Augustus</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>PC 17569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1215</th>\n",
       "      <td>33.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>26.5500</td>\n",
       "      <td>Rowe, Mr. Alfred G</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>113790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1219</th>\n",
       "      <td>46.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>79.2000</td>\n",
       "      <td>Rosenshine, Mr. George (Mr George Thorne\")\"</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>PC 17585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1223</th>\n",
       "      <td>39.0</td>\n",
       "      <td>A18</td>\n",
       "      <td>C</td>\n",
       "      <td>29.7000</td>\n",
       "      <td>Dulles, Mr. William Crothers</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>PC 17580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1227</th>\n",
       "      <td>30.0</td>\n",
       "      <td>C106</td>\n",
       "      <td>S</td>\n",
       "      <td>26.0000</td>\n",
       "      <td>Maguire, Mr. John Edward</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>110469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1247</th>\n",
       "      <td>50.0</td>\n",
       "      <td>E60</td>\n",
       "      <td>S</td>\n",
       "      <td>26.0000</td>\n",
       "      <td>Julian, Mr. Henry Forbes</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>113044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1264</th>\n",
       "      <td>49.0</td>\n",
       "      <td>B52 B54 B56</td>\n",
       "      <td>S</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>Ismay, Mr. Joseph Bruce</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>112058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1270</th>\n",
       "      <td>55.0</td>\n",
       "      <td>C39</td>\n",
       "      <td>S</td>\n",
       "      <td>50.0000</td>\n",
       "      <td>Hipkins, Mr. William Edward</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1282</th>\n",
       "      <td>23.0</td>\n",
       "      <td>B24</td>\n",
       "      <td>S</td>\n",
       "      <td>93.5000</td>\n",
       "      <td>Payne, Mr. Vivian Ponsonby</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>12749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1295</th>\n",
       "      <td>17.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>47.1000</td>\n",
       "      <td>Carrau, Mr. Jose Pedro</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>-888</td>\n",
       "      <td>113059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1296</th>\n",
       "      <td>43.0</td>\n",
       "      <td>D40</td>\n",
       "      <td>C</td>\n",
       "      <td>27.7208</td>\n",
       "      <td>Frauenthal, Mr. Isaac Gerald</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>17765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1299</th>\n",
       "      <td>50.0</td>\n",
       "      <td>C80</td>\n",
       "      <td>C</td>\n",
       "      <td>211.5000</td>\n",
       "      <td>Widener, Mr. George Dunton</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>1</td>\n",
       "      <td>-888</td>\n",
       "      <td>113503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>179 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              Age        Cabin Embarked      Fare  \\\n",
       "PassengerId                                         \n",
       "7            54.0          E46        S   51.8625   \n",
       "24           28.0           A6        S   35.5000   \n",
       "28           19.0  C23 C25 C27        S  263.0000   \n",
       "31           40.0          NaN        C   27.7208   \n",
       "35           28.0          NaN        C   82.1708   \n",
       "36           42.0          NaN        S   52.0000   \n",
       "55           65.0          B30        C   61.9792   \n",
       "56            NaN          C52        S   35.5000   \n",
       "63           45.0          C83        S   83.4750   \n",
       "65            NaN          NaN        C   27.7208   \n",
       "84           28.0          NaN        S   47.1000   \n",
       "93           46.0          E31        S   61.1750   \n",
       "97           71.0           A5        C   34.6542   \n",
       "98           23.0      D10 D12        C   63.3583   \n",
       "103          21.0          D26        S   77.2875   \n",
       "111          47.0         C110        S   52.0000   \n",
       "119          24.0      B58 B60        C  247.5208   \n",
       "125          54.0          D26        S   77.2875   \n",
       "138          37.0         C123        S   53.1000   \n",
       "140          24.0          B86        C   79.2000   \n",
       "156          51.0          NaN        C   61.3792   \n",
       "169           NaN          NaN        S   25.9250   \n",
       "171          61.0          B19        S   33.5000   \n",
       "175          56.0           A7        C   30.6958   \n",
       "186           NaN          A32        S   50.0000   \n",
       "188          45.0          NaN        S   26.5500   \n",
       "210          40.0          A31        C   31.0000   \n",
       "225          38.0          C93        S   90.0000   \n",
       "246          44.0          C78        Q   90.0000   \n",
       "249          37.0          D35        S   52.5542   \n",
       "...           ...          ...      ...       ...   \n",
       "1088          6.0          E34        C  134.5000   \n",
       "1094         47.0      C62 C64        C  227.5250   \n",
       "1097          NaN          NaN        C   25.7417   \n",
       "1107         42.0          B11        S   42.5000   \n",
       "1109         57.0          NaN        S  164.8667   \n",
       "1126         39.0          C85        C   71.2833   \n",
       "1128         64.0          D37        C   75.2500   \n",
       "1134         45.0          E34        C  134.5000   \n",
       "1137         41.0          D21        S   51.8625   \n",
       "1144         27.0          C89        C  136.7792   \n",
       "1158          NaN          NaN        S    0.0000   \n",
       "1162         46.0           C6        C   75.2417   \n",
       "1179         24.0          B45        S   82.2667   \n",
       "1182          NaN          NaN        S   39.6000   \n",
       "1185         53.0          A34        S   81.8583   \n",
       "1190         30.0          NaN        S   45.5000   \n",
       "1198         30.0      C22 C26        S  151.5500   \n",
       "1200         55.0          B69        S   93.5000   \n",
       "1208         57.0          B78        C  146.5208   \n",
       "1215         33.0          NaN        S   26.5500   \n",
       "1219         46.0          NaN        C   79.2000   \n",
       "1223         39.0          A18        C   29.7000   \n",
       "1227         30.0         C106        S   26.0000   \n",
       "1247         50.0          E60        S   26.0000   \n",
       "1264         49.0  B52 B54 B56        S    0.0000   \n",
       "1270         55.0          C39        S   50.0000   \n",
       "1282         23.0          B24        S   93.5000   \n",
       "1295         17.0          NaN        S   47.1000   \n",
       "1296         43.0          D40        C   27.7208   \n",
       "1299         50.0          C80        C  211.5000   \n",
       "\n",
       "                                                      Name  Parch  Pclass  \\\n",
       "PassengerId                                                                 \n",
       "7                                  McCarthy, Mr. Timothy J      0       1   \n",
       "24                            Sloper, Mr. William Thompson      0       1   \n",
       "28                          Fortune, Mr. Charles Alexander      2       1   \n",
       "31                                Uruchurtu, Don. Manuel E      0       1   \n",
       "35                                 Meyer, Mr. Edgar Joseph      0       1   \n",
       "36                          Holverson, Mr. Alexander Oskar      0       1   \n",
       "55                          Ostby, Mr. Engelhart Cornelius      1       1   \n",
       "56                                       Woolner, Mr. Hugh      0       1   \n",
       "63                             Harris, Mr. Henry Birkhardt      0       1   \n",
       "65                                   Stewart, Mr. Albert A      0       1   \n",
       "84                                 Carrau, Mr. Francisco M      0       1   \n",
       "93                             Chaffee, Mr. Herbert Fuller      0       1   \n",
       "97                               Goldschmidt, Mr. George B      0       1   \n",
       "98                         Greenfield, Mr. William Bertram      1       1   \n",
       "103                              White, Mr. Richard Frasar      1       1   \n",
       "111                         Porter, Mr. Walter Chamberlain      0       1   \n",
       "119                               Baxter, Mr. Quigg Edmond      1       1   \n",
       "125                            White, Mr. Percival Wayland      1       1   \n",
       "138                            Futrelle, Mr. Jacques Heath      0       1   \n",
       "140                                     Giglio, Mr. Victor      0       1   \n",
       "156                            Williams, Mr. Charles Duane      1       1   \n",
       "169                                    Baumann, Mr. John D      0       1   \n",
       "171                              Van der hoef, Mr. Wyckoff      0       1   \n",
       "175                                Smith, Mr. James Clinch      0       1   \n",
       "186                                  Rood, Mr. Hugh Roscoe      0       1   \n",
       "188          Romaine, Mr. Charles Hallace (\"Mr C Rolmane\")      0       1   \n",
       "210                                       Blank, Mr. Henry      0       1   \n",
       "225                           Hoyt, Mr. Frederick Maxfield      0       1   \n",
       "246                            Minahan, Dr. William Edward      0       1   \n",
       "249                          Beckwith, Mr. Richard Leonard      1       1   \n",
       "...                                                    ...    ...     ...   \n",
       "1088                       Spedden, Master. Robert Douglas      2       1   \n",
       "1094                                Astor, Col. John Jacob      0       1   \n",
       "1097                             Omont, Mr. Alfred Fernand      0       1   \n",
       "1107                                 Head, Mr. Christopher      0       1   \n",
       "1109                              Wick, Mr. George Dennick      1       1   \n",
       "1126                             Cumings, Mr. John Bradley      0       1   \n",
       "1128                              Warren, Mr. Frank Manley      0       1   \n",
       "1134                          Spedden, Mr. Frederic Oakley      1       1   \n",
       "1137                               Kenyon, Mr. Frederick R      0       1   \n",
       "1144                              Clark, Mr. Walter Miller      0       1   \n",
       "1158                 Chisholm, Mr. Roderick Robert Crispin      0       1   \n",
       "1162                          McCaffry, Mr. Thomas Francis      0       1   \n",
       "1179                            Snyder, Mr. John Pillsbury      0       1   \n",
       "1182                   Rheims, Mr. George Alexander Lucien      0       1   \n",
       "1185                                 Dodge, Dr. Washington      1       1   \n",
       "1190                            Loring, Mr. Joseph Holland      0       1   \n",
       "1198                  Allison, Mr. Hudson Joshua Creighton      2       1   \n",
       "1200                            Hays, Mr. Charles Melville      1       1   \n",
       "1208                         Spencer, Mr. William Augustus      0       1   \n",
       "1215                                    Rowe, Mr. Alfred G      0       1   \n",
       "1219           Rosenshine, Mr. George (Mr George Thorne\")\"      0       1   \n",
       "1223                          Dulles, Mr. William Crothers      0       1   \n",
       "1227                              Maguire, Mr. John Edward      0       1   \n",
       "1247                              Julian, Mr. Henry Forbes      0       1   \n",
       "1264                               Ismay, Mr. Joseph Bruce      0       1   \n",
       "1270                           Hipkins, Mr. William Edward      0       1   \n",
       "1282                            Payne, Mr. Vivian Ponsonby      0       1   \n",
       "1295                                Carrau, Mr. Jose Pedro      0       1   \n",
       "1296                          Frauenthal, Mr. Isaac Gerald      0       1   \n",
       "1299                            Widener, Mr. George Dunton      1       1   \n",
       "\n",
       "              Sex  SibSp  Survived       Ticket  \n",
       "PassengerId                                      \n",
       "7            male      0         0        17463  \n",
       "24           male      0         1       113788  \n",
       "28           male      3         0        19950  \n",
       "31           male      0         0     PC 17601  \n",
       "35           male      1         0     PC 17604  \n",
       "36           male      1         0       113789  \n",
       "55           male      0         0       113509  \n",
       "56           male      0         1        19947  \n",
       "63           male      1         0        36973  \n",
       "65           male      0         0     PC 17605  \n",
       "84           male      0         0       113059  \n",
       "93           male      1         0  W.E.P. 5734  \n",
       "97           male      0         0     PC 17754  \n",
       "98           male      0         1     PC 17759  \n",
       "103          male      0         0        35281  \n",
       "111          male      0         0       110465  \n",
       "119          male      0         0     PC 17558  \n",
       "125          male      0         0        35281  \n",
       "138          male      1         0       113803  \n",
       "140          male      0         0     PC 17593  \n",
       "156          male      0         0     PC 17597  \n",
       "169          male      0         0     PC 17318  \n",
       "171          male      0         0       111240  \n",
       "175          male      0         0        17764  \n",
       "186          male      0         0       113767  \n",
       "188          male      0         1       111428  \n",
       "210          male      0         1       112277  \n",
       "225          male      1         1        19943  \n",
       "246          male      2         0        19928  \n",
       "249          male      1         1        11751  \n",
       "...           ...    ...       ...          ...  \n",
       "1088         male      0      -888        16966  \n",
       "1094         male      1      -888     PC 17757  \n",
       "1097         male      0      -888   F.C. 12998  \n",
       "1107         male      0      -888       113038  \n",
       "1109         male      1      -888        36928  \n",
       "1126         male      1      -888     PC 17599  \n",
       "1128         male      1      -888       110813  \n",
       "1134         male      1      -888        16966  \n",
       "1137         male      1      -888        17464  \n",
       "1144         male      1      -888        13508  \n",
       "1158         male      0      -888       112051  \n",
       "1162         male      0      -888        13050  \n",
       "1179         male      1      -888        21228  \n",
       "1182         male      0      -888     PC 17607  \n",
       "1185         male      1      -888        33638  \n",
       "1190         male      0      -888       113801  \n",
       "1198         male      1      -888       113781  \n",
       "1200         male      1      -888        12749  \n",
       "1208         male      1      -888     PC 17569  \n",
       "1215         male      0      -888       113790  \n",
       "1219         male      0      -888     PC 17585  \n",
       "1223         male      0      -888     PC 17580  \n",
       "1227         male      0      -888       110469  \n",
       "1247         male      0      -888       113044  \n",
       "1264         male      0      -888       112058  \n",
       "1270         male      0      -888          680  \n",
       "1282         male      0      -888        12749  \n",
       "1295         male      0      -888       113059  \n",
       "1296         male      1      -888        17765  \n",
       "1299         male      1      -888       113503  \n",
       "\n",
       "[179 rows x 11 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SELECT statements to display rows\n",
    "df.info()\n",
    "df.head(10)\n",
    "df.tail(10)\n",
    "df['Name']\n",
    "df[['Name', 'Age']]\n",
    "df.loc[0:5]\n",
    "df.loc[0:5,['Name','Age']]\n",
    "df.loc[0:5,'Name':'Age']\n",
    "df.iloc[0:5,1:3]\n",
    "\n",
    "#SELECT statements with a WHERE clause\n",
    "df[df.Embarked.isnull()]\n",
    "df['Sex'].unique()\n",
    "df[df.Sex == 'male']\n",
    "df[(df.Sex =='male') & (df.Pclass == 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Name</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Ticket</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1046.000000</td>\n",
       "      <td>295</td>\n",
       "      <td>1307</td>\n",
       "      <td>1308.000000</td>\n",
       "      <td>1309</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>NaN</td>\n",
       "      <td>186</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1307</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>NaN</td>\n",
       "      <td>C23 C25 C27</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Kelly, Mr. James</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>male</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CA. 2343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>914</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>843</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>29.881138</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33.295479</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.385027</td>\n",
       "      <td>2.294882</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.498854</td>\n",
       "      <td>-283.301757</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>14.413493</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>51.758668</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.865560</td>\n",
       "      <td>0.837836</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.041658</td>\n",
       "      <td>414.337413</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.170000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-888.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>21.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.895800</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-888.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>28.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.454200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>39.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31.275000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>80.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.329200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Age        Cabin Embarked         Fare              Name  \\\n",
       "count   1046.000000          295     1307  1308.000000              1309   \n",
       "unique          NaN          186        3          NaN              1307   \n",
       "top             NaN  C23 C25 C27        S          NaN  Kelly, Mr. James   \n",
       "freq            NaN            6      914          NaN                 2   \n",
       "mean      29.881138          NaN      NaN    33.295479               NaN   \n",
       "std       14.413493          NaN      NaN    51.758668               NaN   \n",
       "min        0.170000          NaN      NaN     0.000000               NaN   \n",
       "25%       21.000000          NaN      NaN     7.895800               NaN   \n",
       "50%       28.000000          NaN      NaN    14.454200               NaN   \n",
       "75%       39.000000          NaN      NaN    31.275000               NaN   \n",
       "max       80.000000          NaN      NaN   512.329200               NaN   \n",
       "\n",
       "              Parch       Pclass   Sex        SibSp     Survived    Ticket  \n",
       "count   1309.000000  1309.000000  1309  1309.000000  1309.000000      1309  \n",
       "unique          NaN          NaN     2          NaN          NaN       929  \n",
       "top             NaN          NaN  male          NaN          NaN  CA. 2343  \n",
       "freq            NaN          NaN   843          NaN          NaN        11  \n",
       "mean       0.385027     2.294882   NaN     0.498854  -283.301757       NaN  \n",
       "std        0.865560     0.837836   NaN     1.041658   414.337413       NaN  \n",
       "min        0.000000     1.000000   NaN     0.000000  -888.000000       NaN  \n",
       "25%        0.000000     2.000000   NaN     0.000000  -888.000000       NaN  \n",
       "50%        0.000000     3.000000   NaN     0.000000     0.000000       NaN  \n",
       "75%        0.000000     3.000000   NaN     1.000000     1.000000       NaN  \n",
       "max        9.000000     3.000000   NaN     8.000000     1.000000       NaN  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#STATISTICS for numerical Data\n",
    "df.describe()\n",
    "#df.Fare.describe()\n",
    "df.describe(include='all')\n",
    "\n",
    "#Counting values and Percentages\n",
    "df['Sex'].value_counts() #you can also do df.Sex.value_counts()\n",
    "df['Sex'].value_counts(normalize=True)\n",
    "df[df.Sex != 'male'].Sex.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DATA VISUALIZATION. Semi-colon gets rid of useless label\n",
    "df.Fare.skew() # Gives you the skewness of FARE. \n",
    "\n",
    "df.Fare.plot(kind='box') #Visualize the Fare as a box whisker plot\n",
    "\n",
    "df.Age.plot(kind='hist', title = 'histogram for ages', color='r', bins=20)\n",
    "# value_counts gives the actual  count of numerical? white creating a hist pandas will neglect all missing values\n",
    "\n",
    "df.Pclass.value_counts().plot(kind='bar', rot=0, title='Class wise Passenger count', color='c'); \n",
    "# rot =0 makes the labels verticle. title gives it a title, color gives it a color. The semi colon is important i think??\n",
    "\n",
    "df.Age.plot(kind='kde', title = 'Density plot for Age', color = 'c');\n",
    "\n",
    "df.plot.scatter(x='Age', y='Fare', color='c', title='Scatter Plot of Age vs Fare', alpha=0.25);\n",
    "# the alpha shows opacity of dots. Scatter plot requires an X and a Y\n",
    "\n",
    "df[df.Age.notnull()].boxplot('Age','Sex'); # Boxplot on Age v Sex\n",
    "\n",
    "pd.crosstab(df.Sex, df.Pclass).plot(kind='bar'); #Graphing a crosstab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#GROUPBY TO FIND AGGREGATE VALUES    (Categorical)[Numeric] <-- Format is like this\n",
    "df.groupby('Sex')['Age'].median() #GroupBy Sex, Find Median of Age of those grouped (just like sql)\n",
    "#df.groupby('Sex').Age.median()\n",
    "\n",
    "df.groupby('Pclass')['Fare', 'Age'].median() # Gives you the median on multiple results\n",
    "\n",
    "df.groupby(['Pclass', 'Embarked'])['Fare'].median() #GroupBy multiple columns. Median Fare for Embarked and Pclass\n",
    "\n",
    "df.groupby('Pclass').agg({'Fare': 'mean', 'Age': 'median'})# Allows you to use multiple aggregate functions. \n",
    "\n",
    "#CROSSTAB\n",
    "pd.crosstab(df.Sex, df.Pclass) #Crosstab combines the information from multiple sources. 2x3=6\n",
    "pd.crosstab(df[df.Survived != -888].Survived, df[df.Embarked != 'C'].Embarked) #More complex CrossTab\n",
    "\n",
    "#PIVOT TABLES [Same as Groupby]\n",
    "df.pivot_table(index='Sex', columns='Pclass',values='Age', aggfunc='mean') #pivot tables are useful when working on numerical feature for different categorical features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################DATA WRANGLING / DATA MUNGING STARTING THE NEXT CELL#############################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df.Fare.isnull()] # df.info() and we see that Fare has null value so let's fill it in\n",
    "#df.groupby(['Pclass','Embarked'])['Fare'].median() <-- This gives you the groupby table, dont use this\n",
    "medianfare = df.loc[(df.Pclass == 3) & (df.Embarked == 'S'),'Fare'].median() # Use this. This gives you VALUE of 8.05\n",
    "df.Fare.fillna(medianfare, inplace= True) #Now it's filled in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GetTitle(name): # When we df.info(), we want to get value from name. Feature Engineer a title from name. (payment-identifier)\n",
    "    firstnamewithtitle = name.split(',')[1] #take 'name' and splits by the comma. everything after the 'comma' is put here bc [1]\n",
    "    title = firstnamewithtitle.split('.')[0]#takes the upper part and splits again by the '.' [0] gives you the title\n",
    "    title = title.strip().lower() # strip removes all white spaces, and the .lower makes it all lower case.\n",
    "    return title # return the title. Now the GetTitle function will give you the title for each Name\n",
    "df['Title'] = df.Name.map(lambda x : GetTitle(x)) #This is how you add a feature in the pandas dataframe (INSERT)!!!!!!!\n",
    "\n",
    "#df[df.Age.notnull()].boxplot('Age','Title'); #<-- you can see it graphed, and see that Title is a good candidate to predict Age\n",
    "title_age_median = df.groupby('Title').Age.transform('median') #Get median Age for Title and put into variable, title_age_median\n",
    "df.Age.fillna(title_age_median , inplace=True) #put title_age_median into all Nulls for Age."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### OPTIONAL THIS IS DEALING WITH OUTLIERS. USING LOGS###\n",
    "### OPTIONAL THIS IS DEALING WITH OUTLIERS. USING LOGS###\n",
    "### OPTIONAL THIS IS DEALING WITH OUTLIERS. USING LOGS###\n",
    "\n",
    "#OUTLIERS & TRANSFORMATIONS. Regarding Outliers, we can find them using Histogram, BoxPlot, or a Scatterplot Graph.\n",
    "df.Age.plot(kind='hist', bins = 20, color = 'c'); #most age is in 20-30's, but few outliers in the 70's and 80's\n",
    "df.loc[df.Age > 70] # Let's check out the rows where age is more than 70. Here we see 6 people\n",
    "df.loc[df.Fare == df.Fare.max()] #Fare has a lot of outliers too when graphed, so let's seethe max fare\n",
    "\n",
    "# Since we have a skewed dsitribution for Fare.We can apply a TRANSFORMATION to make it less skewed. Transformation is a common \n",
    "# way to treat outliers to some extent. Since we know passenger Fare will never be negative we can apply the Log Transformation.\n",
    "Logfare = np.log(df.Fare + 1.0) # We add 1 to the fare to accomidate $0 Fares. This is bc log(0) is undefined."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FEATURE ENGINEERING [BINNING]. We cut the Fare into 4 bins, where each bin contains an EQUAL number of observations\n",
    "df['Fare_Bin'] = pd.qcut(df.Fare, 4, labels=['very_low', 'low', 'high', 'very_high'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### OPTIONAL SELF MADE PRACTICE \n",
    "def selfmade(fare):\n",
    "    if fare <= 100:\n",
    "        return '<= 100'\n",
    "    elif fare > 100 and fare <= 200:\n",
    "        return '100 < fare <= 200'\n",
    "    elif fare > 200 and fare <= 300:\n",
    "        return '200 < fare <= 300'\n",
    "    else:\n",
    "        return '300 < fare'\n",
    "    \n",
    "df['SelfMade'] = df.Fare.map(lambda x : selfmade(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LEFT OFF HERE PRACTICE FROM HERE ON!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Feature Engineering Adult/Minor\n",
    "df['AgeState'] = np.where(df['Age'] >= 18, 'Adult', 'Child') #If True then 'Adult', Else 'Child'\n",
    "df['AgeState'].value_counts() # Now let's count them\n",
    "pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].AgeState) #crosstab survival between Adults and Chilren\n",
    "\n",
    "#Checking if Bigger Family means higher survival rate. First we INSERT columns adding Parch(parents) + SibSp(Siblings) + 1(self)\n",
    "df['FamilySize'] = df.Parch + df.SibSp + 1 \n",
    "df['FamilySize'].plot(kind='hist', color='c'); # then we graph it to see the distribution. We see some Outliers\n",
    "df.loc[df.FamilySize == df.FamilySize.max()] #Let's see what the max value of family size is.\n",
    "df.loc[df.FamilySize == df.FamilySize.max(),['Name','Survived','FamilySize','Ticket']] # Same as above, but with SELECT\n",
    "pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].FamilySize) # Checking with all Familysizes\n",
    "\n",
    "#Let's see if motherhood plays an impact on survival\n",
    "df['IsMother'] = np.where(((df.Sex == 'female') & (df.Parch > 0) & (df.Age > 18) & (df.Title != 'Miss')),1,0) #If true 1, else 0\n",
    "pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].IsMother) # Here see see motherhood does affect survival\n",
    "\n",
    "#Let's see if Cabin placement affects survival\n",
    "df.Cabin.unique() # Shows all unique cabins\n",
    "df.loc[df.Cabin == 'T'] # We extract all rows where Cabin value is 'T', and see that the T was prolly a typo cause only 1 value\n",
    "df.loc[df.Cabin == 'T', 'Cabin'] = np.NaN #<-- run this to update T to Naan\n",
    "\n",
    "def get_deck(cabin):\n",
    "    return np.where(pd.notnull(cabin),str(cabin)[0].upper(),'z') #First letter of Cabin. If Null then 'Z'. We convert the cabin \n",
    "                                                        # value to a str() and pull the first letter of it, and capitalize it.\n",
    "\n",
    "df['Deck'] = df['Cabin'].map(lambda x : get_deck(x)) # Map Cabin into get_deck using lambda and feature engineer that into 'Deck'\n",
    "df.Deck.value_counts() #We see that Naan(Z) is most poular followed by C.\n",
    "pd.crosstab(df[df.Survived != -888].Survived, df[df.Survived != -888].Deck) #We see that Deck is a good indicator of survival too"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Binary Encoding [0,1 for Y,N]\n",
    "# Label Encoding [Label Encoding 1,2,3 Low med high] \n",
    "# One hot Encoding [make the grid is_a,is_b, is_c]\n",
    "df['Ismale'] = np.where(df.Sex == 'male', 1, 0) #INSERT 1 if male, else 0\n",
    "\n",
    "df = pd.get_dummies(df,columns=['Deck', 'Pclass', 'Title', 'Fare_Bin', 'Embarked'])\n",
    "# The get_dummies is VERY IMPORTANT. You can pass your cateogrical/numeric features and it gets inserted into df. If you print \n",
    "# -it will now show the one hot encoding. The get_dummies will create a feature for every possible value that exists. if 10 \n",
    "# -titles, it will creat 10 features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DROPPING COLUMNS IN PYTHON\n",
    "df.drop(['AgeState'], axis=1, inplace=True) #Pass column name to remove from data. Axis=1 when dropping. inplace has to be true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  \n",
    "# ."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}