extract_words scratchpad.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "words = pd.read_csv(\"word_list.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>freq</th>\n",
       "      <th>syl</th>\n",
       "      <th>let</th>\n",
       "      <th>word</th>\n",
       "      <th>imag</th>\n",
       "      <th>conc</th>\n",
       "      <th>meaning</th>\n",
       "      <th>list</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>lip</td>\n",
       "      <td>6.57</td>\n",
       "      <td>6.93</td>\n",
       "      <td>5.32</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>nursery</td>\n",
       "      <td>6.10</td>\n",
       "      <td>6.45</td>\n",
       "      <td>7.40</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>ink</td>\n",
       "      <td>6.37</td>\n",
       "      <td>6.87</td>\n",
       "      <td>6.96</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>evidence</td>\n",
       "      <td>3.23</td>\n",
       "      <td>3.45</td>\n",
       "      <td>6.20</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>limelight</td>\n",
       "      <td>4.83</td>\n",
       "      <td>3.06</td>\n",
       "      <td>4.72</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   freq  syl  let       word  imag  conc  meaning  list  id\n",
       "0   100    1    3        lip  6.57  6.93     5.32     1   1\n",
       "1    19    3    7    nursery  6.10  6.45     7.40     1   2\n",
       "2    20    1    3        ink  6.37  6.87     6.96     1   3\n",
       "3    50    3    8   evidence  3.23  3.45     6.20     1   4\n",
       "4     1    2    9  limelight  4.83  3.06     4.72     1   5"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total words: 898\n"
     ]
    }
   ],
   "source": [
    "n = len(words)\n",
    "print \"total words:\" , n\n",
    "\n",
    "words_240 = np.random.choice(words[\"id\"],size=240, replace=False)\n",
    "words_120 = np.random.choice(words_240,size=120, replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[404 160 165  90 767 539 373 192 284 368 399  48 180 121  38 579 662 801\n",
      " 162  66  69 297  45 589 123 571 184 763 847 153 242 236 338 881 619 615\n",
      " 700 599 641 654 723 370 451 102 334 473 649 145  82 350 298 498 697  20\n",
      " 333 869 424 839 380 147 354 482 672 675 705 855 435 444 222 224 235 245\n",
      " 640 526 593 255 717 681 166 131 317 580 870 814 549 878 608  70 487 578\n",
      " 157  59 688 199 747 246 695 486 168  92  17 500  19 256 130  42 283 481\n",
      " 758 172 420  73 739 630 872 335 507 704 588 321 575 177 100 122 390 220\n",
      " 410 393  75 798 891 595 543 141 442 239 494 254 369 663 856 471  61 833\n",
      " 194 348 794 859 252 658 214 759  51 582 709 898 585 581 653 115 238 379\n",
      " 495 555 468 810 357 731 418 719 699 566 851 148 703  26 270 715 231  34\n",
      " 465   5 642 391  29 119 548 206 360 563 332 680 875 489 645 775 217 586\n",
      " 827 346 564 587 340  56 762 295 721 516  80 173  57 601 387 735 743   3\n",
      " 477 423 868 416 737  77 635 113 733 574   2 437 344 741 381 756  54 475\n",
      " 510 201 417 466 296 225]\n",
      "\n",
      "[870 465 444 147 192 703 477 238 763 741 437 360 539 510 184 869 417 373\n",
      " 697 681 601 357  57 141 715 588 390 543 672  92 399  77 827 121 487  19\n",
      " 801 564 739  29 481 498 482 298 767 387 859 348 284 495 256 344 580  80\n",
      " 391 641 868 700 471 737 878 424 317 225 473 231 222  38 680 688 587 833\n",
      " 165 855 468 295 157 585 881 333 370 608 794 675 654 177 283 555 735 704\n",
      " 354  69   2 743 847 410 758 340 321 420 393 759  82 119 172 236 418 775\n",
      "  61 589 699 733 180 839 756 709 369 856 246 575]\n"
     ]
    }
   ],
   "source": [
    "print words_240\n",
    "print\n",
    "print words_120\n",
    "\n",
    "final = words[words[\"id\"].isin(words_240)].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "final[\"is_shown\"] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>freq</th>\n",
       "      <th>syl</th>\n",
       "      <th>let</th>\n",
       "      <th>word</th>\n",
       "      <th>imag</th>\n",
       "      <th>conc</th>\n",
       "      <th>meaning</th>\n",
       "      <th>list</th>\n",
       "      <th>id</th>\n",
       "      <th>is_shown</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>nursery</td>\n",
       "      <td>6.10</td>\n",
       "      <td>6.45</td>\n",
       "      <td>7.40</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>ink</td>\n",
       "      <td>6.37</td>\n",
       "      <td>6.87</td>\n",
       "      <td>6.96</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>limelight</td>\n",
       "      <td>4.83</td>\n",
       "      <td>3.06</td>\n",
       "      <td>4.72</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>immunity</td>\n",
       "      <td>3.43</td>\n",
       "      <td>2.46</td>\n",
       "      <td>5.76</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>obsession</td>\n",
       "      <td>3.87</td>\n",
       "      <td>1.65</td>\n",
       "      <td>4.16</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    freq  syl  let       word  imag  conc  meaning  list  id  is_shown\n",
       "1     19    3    7    nursery  6.10  6.45     7.40     1   2         0\n",
       "2     20    1    3        ink  6.37  6.87     6.96     1   3         0\n",
       "4      1    2    9  limelight  4.83  3.06     4.72     1   5         0\n",
       "16     3    4    8   immunity  3.43  2.46     5.76     1  17         0\n",
       "18     2    3    9  obsession  3.87  1.65     4.16     1  19         0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ids = list(final[\"id\"])\n",
    "is_shown = list(final[\"is_shown\"])\n",
    "\n",
    "for x in range(0,239):\n",
    "    #print x, ids[x], is_shown[x]\n",
    "    if ids[x] in words_120:\n",
    "        is_shown[x] = 1\n",
    "\n",
    "final[\"is_shown\"] = is_shown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "240"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(is_shown)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(is_shown)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>freq</th>\n",
       "      <th>syl</th>\n",
       "      <th>let</th>\n",
       "      <th>word</th>\n",
       "      <th>imag</th>\n",
       "      <th>conc</th>\n",
       "      <th>meaning</th>\n",
       "      <th>list</th>\n",
       "      <th>id</th>\n",
       "      <th>is_shown</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>nursery</td>\n",
       "      <td>6.10</td>\n",
       "      <td>6.45</td>\n",
       "      <td>7.40</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>ink</td>\n",
       "      <td>6.37</td>\n",
       "      <td>6.87</td>\n",
       "      <td>6.96</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>limelight</td>\n",
       "      <td>4.83</td>\n",
       "      <td>3.06</td>\n",
       "      <td>4.72</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>immunity</td>\n",
       "      <td>3.43</td>\n",
       "      <td>2.46</td>\n",
       "      <td>5.76</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>obsession</td>\n",
       "      <td>3.87</td>\n",
       "      <td>1.65</td>\n",
       "      <td>4.16</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>horsehair</td>\n",
       "      <td>5.67</td>\n",
       "      <td>6.80</td>\n",
       "      <td>6.20</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>charlatan</td>\n",
       "      <td>3.43</td>\n",
       "      <td>5.28</td>\n",
       "      <td>3.19</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>exactitude</td>\n",
       "      <td>2.23</td>\n",
       "      <td>1.69</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>profession</td>\n",
       "      <td>3.83</td>\n",
       "      <td>3.65</td>\n",
       "      <td>5.44</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>23</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>drama</td>\n",
       "      <td>4.90</td>\n",
       "      <td>3.66</td>\n",
       "      <td>7.00</td>\n",
       "      <td>1</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>reminder</td>\n",
       "      <td>2.83</td>\n",
       "      <td>3.48</td>\n",
       "      <td>5.20</td>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>fun</td>\n",
       "      <td>5.23</td>\n",
       "      <td>2.45</td>\n",
       "      <td>5.50</td>\n",
       "      <td>1</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>occasion</td>\n",
       "      <td>2.53</td>\n",
       "      <td>3.22</td>\n",
       "      <td>5.00</td>\n",
       "      <td>1</td>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>truth</td>\n",
       "      <td>2.73</td>\n",
       "      <td>1.69</td>\n",
       "      <td>4.78</td>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>coffee</td>\n",
       "      <td>6.73</td>\n",
       "      <td>6.89</td>\n",
       "      <td>7.28</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>robbery</td>\n",
       "      <td>5.00</td>\n",
       "      <td>4.75</td>\n",
       "      <td>6.16</td>\n",
       "      <td>1</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>panic</td>\n",
       "      <td>5.33</td>\n",
       "      <td>2.18</td>\n",
       "      <td>6.72</td>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>29</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>substitute</td>\n",
       "      <td>3.13</td>\n",
       "      <td>4.56</td>\n",
       "      <td>4.48</td>\n",
       "      <td>1</td>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>chin</td>\n",
       "      <td>6.43</td>\n",
       "      <td>6.96</td>\n",
       "      <td>5.28</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>phantom</td>\n",
       "      <td>5.03</td>\n",
       "      <td>2.50</td>\n",
       "      <td>5.00</td>\n",
       "      <td>1</td>\n",
       "      <td>66</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>32</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>barrel</td>\n",
       "      <td>6.57</td>\n",
       "      <td>6.94</td>\n",
       "      <td>6.16</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>essence</td>\n",
       "      <td>2.33</td>\n",
       "      <td>1.66</td>\n",
       "      <td>3.28</td>\n",
       "      <td>1</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>100</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>material</td>\n",
       "      <td>5.07</td>\n",
       "      <td>6.10</td>\n",
       "      <td>5.92</td>\n",
       "      <td>1</td>\n",
       "      <td>73</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>wench</td>\n",
       "      <td>5.30</td>\n",
       "      <td>6.20</td>\n",
       "      <td>5.08</td>\n",
       "      <td>1</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>chasm</td>\n",
       "      <td>4.47</td>\n",
       "      <td>5.25</td>\n",
       "      <td>6.04</td>\n",
       "      <td>1</td>\n",
       "      <td>77</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>elephant</td>\n",
       "      <td>6.83</td>\n",
       "      <td>7.00</td>\n",
       "      <td>6.88</td>\n",
       "      <td>1</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>infirmary</td>\n",
       "      <td>5.63</td>\n",
       "      <td>6.43</td>\n",
       "      <td>6.65</td>\n",
       "      <td>1</td>\n",
       "      <td>82</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>ignorance</td>\n",
       "      <td>3.57</td>\n",
       "      <td>1.75</td>\n",
       "      <td>4.00</td>\n",
       "      <td>1</td>\n",
       "      <td>90</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>26</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>item</td>\n",
       "      <td>3.67</td>\n",
       "      <td>5.27</td>\n",
       "      <td>4.96</td>\n",
       "      <td>1</td>\n",
       "      <td>92</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>grief</td>\n",
       "      <td>4.70</td>\n",
       "      <td>1.86</td>\n",
       "      <td>6.32</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>746</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>shotgun</td>\n",
       "      <td>6.60</td>\n",
       "      <td>6.96</td>\n",
       "      <td>7.88</td>\n",
       "      <td>1</td>\n",
       "      <td>747</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>icebox</td>\n",
       "      <td>6.17</td>\n",
       "      <td>6.90</td>\n",
       "      <td>6.28</td>\n",
       "      <td>1</td>\n",
       "      <td>756</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>757</th>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>fortune</td>\n",
       "      <td>4.57</td>\n",
       "      <td>3.82</td>\n",
       "      <td>6.00</td>\n",
       "      <td>1</td>\n",
       "      <td>758</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>758</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>subtraction</td>\n",
       "      <td>3.80</td>\n",
       "      <td>3.73</td>\n",
       "      <td>5.52</td>\n",
       "      <td>1</td>\n",
       "      <td>759</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>761</th>\n",
       "      <td>50</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>university</td>\n",
       "      <td>6.53</td>\n",
       "      <td>5.87</td>\n",
       "      <td>7.20</td>\n",
       "      <td>1</td>\n",
       "      <td>762</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>762</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>hurdle</td>\n",
       "      <td>6.33</td>\n",
       "      <td>6.65</td>\n",
       "      <td>5.92</td>\n",
       "      <td>1</td>\n",
       "      <td>763</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>32</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>cellar</td>\n",
       "      <td>6.27</td>\n",
       "      <td>6.83</td>\n",
       "      <td>6.79</td>\n",
       "      <td>1</td>\n",
       "      <td>767</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>anxiety</td>\n",
       "      <td>4.03</td>\n",
       "      <td>1.63</td>\n",
       "      <td>5.69</td>\n",
       "      <td>1</td>\n",
       "      <td>775</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793</th>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>painter</td>\n",
       "      <td>5.87</td>\n",
       "      <td>6.59</td>\n",
       "      <td>6.40</td>\n",
       "      <td>1</td>\n",
       "      <td>794</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>29</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>twilight</td>\n",
       "      <td>5.93</td>\n",
       "      <td>5.77</td>\n",
       "      <td>6.56</td>\n",
       "      <td>1</td>\n",
       "      <td>798</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>50</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>potato</td>\n",
       "      <td>6.50</td>\n",
       "      <td>7.00</td>\n",
       "      <td>7.13</td>\n",
       "      <td>1</td>\n",
       "      <td>801</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>809</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>diffusion</td>\n",
       "      <td>3.40</td>\n",
       "      <td>3.18</td>\n",
       "      <td>5.67</td>\n",
       "      <td>1</td>\n",
       "      <td>810</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>813</th>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>boy</td>\n",
       "      <td>6.57</td>\n",
       "      <td>6.93</td>\n",
       "      <td>6.96</td>\n",
       "      <td>1</td>\n",
       "      <td>814</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>826</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>charter</td>\n",
       "      <td>3.83</td>\n",
       "      <td>5.05</td>\n",
       "      <td>4.89</td>\n",
       "      <td>1</td>\n",
       "      <td>827</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>832</th>\n",
       "      <td>22</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>salute</td>\n",
       "      <td>5.77</td>\n",
       "      <td>5.00</td>\n",
       "      <td>6.52</td>\n",
       "      <td>1</td>\n",
       "      <td>833</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>838</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>deluge</td>\n",
       "      <td>4.57</td>\n",
       "      <td>4.38</td>\n",
       "      <td>5.32</td>\n",
       "      <td>1</td>\n",
       "      <td>839</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>unbeliever</td>\n",
       "      <td>2.60</td>\n",
       "      <td>4.00</td>\n",
       "      <td>4.72</td>\n",
       "      <td>1</td>\n",
       "      <td>847</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>850</th>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>house</td>\n",
       "      <td>6.67</td>\n",
       "      <td>6.93</td>\n",
       "      <td>6.83</td>\n",
       "      <td>1</td>\n",
       "      <td>851</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>854</th>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>permission</td>\n",
       "      <td>2.87</td>\n",
       "      <td>4.43</td>\n",
       "      <td>4.68</td>\n",
       "      <td>1</td>\n",
       "      <td>855</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>855</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>hillside</td>\n",
       "      <td>6.30</td>\n",
       "      <td>6.59</td>\n",
       "      <td>6.92</td>\n",
       "      <td>1</td>\n",
       "      <td>856</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858</th>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>sea</td>\n",
       "      <td>6.73</td>\n",
       "      <td>6.79</td>\n",
       "      <td>7.16</td>\n",
       "      <td>1</td>\n",
       "      <td>859</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>867</th>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>headquarters</td>\n",
       "      <td>5.03</td>\n",
       "      <td>5.94</td>\n",
       "      <td>5.08</td>\n",
       "      <td>1</td>\n",
       "      <td>868</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>poster</td>\n",
       "      <td>6.33</td>\n",
       "      <td>6.96</td>\n",
       "      <td>6.20</td>\n",
       "      <td>1</td>\n",
       "      <td>869</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869</th>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>utensil</td>\n",
       "      <td>5.47</td>\n",
       "      <td>6.58</td>\n",
       "      <td>6.08</td>\n",
       "      <td>1</td>\n",
       "      <td>870</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871</th>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>prison</td>\n",
       "      <td>6.23</td>\n",
       "      <td>6.62</td>\n",
       "      <td>7.21</td>\n",
       "      <td>1</td>\n",
       "      <td>872</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>874</th>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>restaurant</td>\n",
       "      <td>6.47</td>\n",
       "      <td>6.83</td>\n",
       "      <td>6.88</td>\n",
       "      <td>1</td>\n",
       "      <td>875</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>877</th>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>vest</td>\n",
       "      <td>6.37</td>\n",
       "      <td>6.73</td>\n",
       "      <td>5.96</td>\n",
       "      <td>1</td>\n",
       "      <td>878</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>880</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>doorman</td>\n",
       "      <td>6.40</td>\n",
       "      <td>6.52</td>\n",
       "      <td>6.28</td>\n",
       "      <td>1</td>\n",
       "      <td>881</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>29</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>beggar</td>\n",
       "      <td>6.40</td>\n",
       "      <td>6.25</td>\n",
       "      <td>6.50</td>\n",
       "      <td>1</td>\n",
       "      <td>891</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>897</th>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>fireplace</td>\n",
       "      <td>6.83</td>\n",
       "      <td>6.96</td>\n",
       "      <td>7.08</td>\n",
       "      <td>1</td>\n",
       "      <td>898</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>240 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     freq  syl  let          word  imag  conc  meaning  list   id  is_shown\n",
       "1      19    3    7       nursery  6.10  6.45     7.40     1    2         1\n",
       "2      20    1    3           ink  6.37  6.87     6.96     1    3         0\n",
       "4       1    2    9     limelight  4.83  3.06     4.72     1    5         0\n",
       "16      3    4    8      immunity  3.43  2.46     5.76     1   17         0\n",
       "18      2    3    9     obsession  3.87  1.65     4.16     1   19         1\n",
       "19      2    2    9     horsehair  5.67  6.80     6.20     1   20         0\n",
       "25      1    3    9     charlatan  3.43  5.28     3.19     1   26         0\n",
       "28      1    4   10    exactitude  2.23  1.69     3.00     1   29         1\n",
       "33     28    3   10    profession  3.83  3.65     5.44     1   34         0\n",
       "37     23    2    5         drama  4.90  3.66     7.00     1   38         1\n",
       "41     17    3    8      reminder  2.83  3.48     5.20     1   42         0\n",
       "44     50    1    3           fun  5.23  2.45     5.50     1   45         0\n",
       "47     50    3    8      occasion  2.53  3.22     5.00     1   48         0\n",
       "50    100    1    5         truth  2.73  1.69     4.78     1   51         0\n",
       "53     50    2    6        coffee  6.73  6.89     7.28     1   54         0\n",
       "55      9    3    7       robbery  5.00  4.75     6.16     1   56         0\n",
       "56     19    2    5         panic  5.33  2.18     6.72     1   57         1\n",
       "58     29    3   10    substitute  3.13  4.56     4.48     1   59         0\n",
       "60     27    1    4          chin  6.43  6.96     5.28     1   61         1\n",
       "65      6    2    7       phantom  5.03  2.50     5.00     1   66         0\n",
       "68     32    2    6        barrel  6.57  6.94     6.16     1   69         1\n",
       "69      8    2    7       essence  2.33  1.66     3.28     1   70         0\n",
       "72    100    4    8      material  5.07  6.10     5.92     1   73         0\n",
       "74      4    1    5         wench  5.30  6.20     5.08     1   75         0\n",
       "76      6    1    5         chasm  4.47  5.25     6.04     1   77         0\n",
       "79     35    3    8      elephant  6.83  7.00     6.88     1   80         0\n",
       "81      1    4    9     infirmary  5.63  6.43     6.65     1   82         0\n",
       "89     22    3    9     ignorance  3.57  1.75     4.00     1   90         0\n",
       "91     26    2    4          item  3.67  5.27     4.96     1   92         0\n",
       "99     45    1    5         grief  4.70  1.86     6.32     1  100         0\n",
       "..    ...  ...  ...           ...   ...   ...      ...   ...  ...       ...\n",
       "746     3    2    7       shotgun  6.60  6.96     7.88     1  747         0\n",
       "755     1    2    6        icebox  6.17  6.90     6.28     1  756         0\n",
       "757    50    2    7       fortune  4.57  3.82     6.00     1  758         0\n",
       "758     1    3   11   subtraction  3.80  3.73     5.52     1  759         0\n",
       "761    50    5   10    university  6.53  5.87     7.20     1  762         0\n",
       "762     2    2    6        hurdle  6.33  6.65     5.92     1  763         0\n",
       "766    32    2    6        cellar  6.27  6.83     6.79     1  767         0\n",
       "774    22    4    7       anxiety  4.03  1.63     5.69     1  775         0\n",
       "793    25    2    7       painter  5.87  6.59     6.40     1  794         0\n",
       "797    29    2    8      twilight  5.93  5.77     6.56     1  798         0\n",
       "800    50    3    6        potato  6.50  7.00     7.13     1  801         0\n",
       "809     2    3    9     diffusion  3.40  3.18     5.67     1  810         0\n",
       "813   100    1    3           boy  6.57  6.93     6.96     1  814         0\n",
       "826    20    2    7       charter  3.83  5.05     4.89     1  827         0\n",
       "832    22    2    6        salute  5.77  5.00     6.52     1  833         0\n",
       "838     2    2    6        deluge  4.57  4.38     5.32     1  839         0\n",
       "846     1    4   10    unbeliever  2.60  4.00     4.72     1  847         0\n",
       "850   100    1    5         house  6.67  6.93     6.83     1  851         0\n",
       "854    22    3   10    permission  2.87  4.43     4.68     1  855         0\n",
       "855    20    2    8      hillside  6.30  6.59     6.92     1  856         0\n",
       "858   100    1    3           sea  6.73  6.79     7.16     1  859         0\n",
       "867    23    3   12  headquarters  5.03  5.94     5.08     1  868         0\n",
       "868     5    2    6        poster  6.33  6.96     6.20     1  869         0\n",
       "869     8    3    7       utensil  5.47  6.58     6.08     1  870         0\n",
       "871    50    2    6        prison  6.23  6.62     7.21     1  872         0\n",
       "874    23    3   10    restaurant  6.47  6.83     6.88     1  875         0\n",
       "877    21    1    4          vest  6.37  6.73     5.96     1  878         0\n",
       "880     1    2    7       doorman  6.40  6.52     6.28     1  881         0\n",
       "890    29    2    6        beggar  6.40  6.25     6.50     1  891         0\n",
       "897    19    2    9     fireplace  6.83  6.96     7.08     1  898         0\n",
       "\n",
       "[240 rows x 10 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}