BasicTemplate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Introduction:\n",
    "\n",
    "In this project, we are going to use stanford NER model to check if the name in GKG dataframe is an organization or not. Here is some basic set-up information, you can do it in your own way.\n",
    "\n",
    "## Basic info and requirement:\n",
    "\n",
    "* Output file should be in __.csv__ file.\n",
    "* Format of output dataframe should contain 2 columns, __original_name, cleaned_name__. Here original_name and cleaned_name are the column from input file, but we will decision by creating a new column which is called __decision__ and we only care about decision == __Yes__. Check this out in the following code.\n",
    "* Each input file contains 500k rows, the generating speed is based on the hardware of your virtual machine, don't run too much rows in one time.(which means you can only take 1000 or 2000 rows and generating the decision)\n",
    "* Save your result all the time since virtual machine might lose connection(You will definitly see this, always save your result after each run)\n",
    "* This jupyter notebook is just a template, which gave you some basic idea how to use these packages, you can write your own loop to generate result."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import boto\n",
    "import boto3\n",
    "import nltk\n",
    "from nltk.tag import StanfordNERTagger\n",
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import java model which we downloaded\n",
    "# check working directory first (in terminal use pwd to check directory, should be fine if your folder name is GKGPreProcessing)\n",
    "st = StanfordNERTagger('/home/ec2-user/GKGPreprocessing/stanford-ner-2018-10-16/classifiers/english.all.3class.distsim.crf.ser.gz',\n",
    "                       '/home/ec2-user/GKGPreprocessing/stanford-ner-2018-10-16/stanford-ner-3.9.2.jar',\n",
    "                       encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/ec2-user/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# you just need download this once.\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# file path which is the place we store input dataframe\n",
    "file_path = 's3://gkgpreprocessing/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set up s3 client and get files inside bucket\n",
    "client = boto3.client('s3')\n",
    "resource = boto3.resource('s3')\n",
    "my_bucket = resource.Bucket('gkgpreprocessing')\n",
    "file_name = []\n",
    "for obj in my_bucket.objects.all():\n",
    "    file_name.append(obj.key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['df2015TO2017_result.csv',\n",
       " 'df_1.csv',\n",
       " 'df_10.csv',\n",
       " 'df_100.csv',\n",
       " 'df_101.csv',\n",
       " 'df_102.csv',\n",
       " 'df_103.csv',\n",
       " 'df_104.csv',\n",
       " 'df_105.csv',\n",
       " 'df_106.csv',\n",
       " 'df_107.csv',\n",
       " 'df_108.csv',\n",
       " 'df_109.csv',\n",
       " 'df_11.csv',\n",
       " 'df_110.csv',\n",
       " 'df_111.csv',\n",
       " 'df_112.csv',\n",
       " 'df_113.csv',\n",
       " 'df_114.csv',\n",
       " 'df_115.csv',\n",
       " 'df_116.csv',\n",
       " 'df_117.csv',\n",
       " 'df_118.csv',\n",
       " 'df_119.csv',\n",
       " 'df_12.csv',\n",
       " 'df_120.csv',\n",
       " 'df_121.csv',\n",
       " 'df_122.csv',\n",
       " 'df_123.csv',\n",
       " 'df_124.csv',\n",
       " 'df_125.csv',\n",
       " 'df_126.csv',\n",
       " 'df_127.csv',\n",
       " 'df_128.csv',\n",
       " 'df_129.csv',\n",
       " 'df_13.csv',\n",
       " 'df_130.csv',\n",
       " 'df_131.csv',\n",
       " 'df_132.csv',\n",
       " 'df_133.csv',\n",
       " 'df_134.csv',\n",
       " 'df_135.csv',\n",
       " 'df_136.csv',\n",
       " 'df_137.csv',\n",
       " 'df_138.csv',\n",
       " 'df_139.csv',\n",
       " 'df_14.csv',\n",
       " 'df_140.csv',\n",
       " 'df_141.csv',\n",
       " 'df_15.csv',\n",
       " 'df_16.csv',\n",
       " 'df_17.csv',\n",
       " 'df_18.csv',\n",
       " 'df_19.csv',\n",
       " 'df_2.csv',\n",
       " 'df_20.csv',\n",
       " 'df_21.csv',\n",
       " 'df_22.csv',\n",
       " 'df_23.csv',\n",
       " 'df_24.csv',\n",
       " 'df_25.csv',\n",
       " 'df_26.csv',\n",
       " 'df_27.csv',\n",
       " 'df_28.csv',\n",
       " 'df_29.csv',\n",
       " 'df_3.csv',\n",
       " 'df_30.csv',\n",
       " 'df_31.csv',\n",
       " 'df_32.csv',\n",
       " 'df_33.csv',\n",
       " 'df_34.csv',\n",
       " 'df_35.csv',\n",
       " 'df_36.csv',\n",
       " 'df_37.csv',\n",
       " 'df_38.csv',\n",
       " 'df_39.csv',\n",
       " 'df_4.csv',\n",
       " 'df_40.csv',\n",
       " 'df_41.csv',\n",
       " 'df_42.csv',\n",
       " 'df_43.csv',\n",
       " 'df_44.csv',\n",
       " 'df_45.csv',\n",
       " 'df_46.csv',\n",
       " 'df_47.csv',\n",
       " 'df_48.csv',\n",
       " 'df_49.csv',\n",
       " 'df_5.csv',\n",
       " 'df_50.csv',\n",
       " 'df_51.csv',\n",
       " 'df_52.csv',\n",
       " 'df_53.csv',\n",
       " 'df_54.csv',\n",
       " 'df_55.csv',\n",
       " 'df_56.csv',\n",
       " 'df_57.csv',\n",
       " 'df_58.csv',\n",
       " 'df_59.csv',\n",
       " 'df_6.csv',\n",
       " 'df_60.csv',\n",
       " 'df_61.csv',\n",
       " 'df_62.csv',\n",
       " 'df_63.csv',\n",
       " 'df_64.csv',\n",
       " 'df_65.csv',\n",
       " 'df_66.csv',\n",
       " 'df_67.csv',\n",
       " 'df_68.csv',\n",
       " 'df_69.csv',\n",
       " 'df_7.csv',\n",
       " 'df_70.csv',\n",
       " 'df_71.csv',\n",
       " 'df_72.csv',\n",
       " 'df_73.csv',\n",
       " 'df_74.csv',\n",
       " 'df_75.csv',\n",
       " 'df_76.csv',\n",
       " 'df_77.csv',\n",
       " 'df_78.csv',\n",
       " 'df_79.csv',\n",
       " 'df_8.csv',\n",
       " 'df_80.csv',\n",
       " 'df_81.csv',\n",
       " 'df_82.csv',\n",
       " 'df_83.csv',\n",
       " 'df_84.csv',\n",
       " 'df_85.csv',\n",
       " 'df_86.csv',\n",
       " 'df_87.csv',\n",
       " 'df_88.csv',\n",
       " 'df_89.csv',\n",
       " 'df_9.csv',\n",
       " 'df_90.csv',\n",
       " 'df_91.csv',\n",
       " 'df_92.csv',\n",
       " 'df_93.csv',\n",
       " 'df_94.csv',\n",
       " 'df_95.csv',\n",
       " 'df_96.csv',\n",
       " 'df_97.csv',\n",
       " 'df_98.csv',\n",
       " 'df_99.csv']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check files\n",
    "file_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Since the first file is the file which contains all the info, we don't need use it.\n",
    "file_name = file_name[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We run a test on df_101.csv\n",
    "new_df = pd.read_csv(file_path + 'df_101.csv',index_col='Unnamed: 0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_name</th>\n",
       "      <th>cleaned_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>series at chipstead sailing clubamerican civil...</td>\n",
       "      <td>series at chipstead sailing clubamerican civil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>serious collision investigation teamap member ...</td>\n",
       "      <td>serious collision investigation teamap member ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>serious crime divisioncamp verde marshal office</td>\n",
       "      <td>serious crime divisioncamp verde marshal office</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>serious crime divisiondepartment of homeland s...</td>\n",
       "      <td>serious crime divisiondepartment of homeland s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>serious crime divisioninternational refugee as...</td>\n",
       "      <td>serious crime divisioninternational refugee as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>serpay folklore group</td>\n",
       "      <td>serpay folklore group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>serta motion essentials adjustable</td>\n",
       "      <td>serta motion essentials adjustable foundation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>service above self registration</td>\n",
       "      <td>service above self registration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>service agent kerry ogrady facing discipline d...</td>\n",
       "      <td>service agent kerry ogrady facing discipline d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>service agreement timatinas biopharma holdings</td>\n",
       "      <td>service agreement timatinas biopharma holdings...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       original_name  \\\n",
       "0  series at chipstead sailing clubamerican civil...   \n",
       "1  serious collision investigation teamap member ...   \n",
       "2    serious crime divisioncamp verde marshal office   \n",
       "3  serious crime divisiondepartment of homeland s...   \n",
       "4  serious crime divisioninternational refugee as...   \n",
       "5                              serpay folklore group   \n",
       "6                 serta motion essentials adjustable   \n",
       "7                    service above self registration   \n",
       "8  service agent kerry ogrady facing discipline d...   \n",
       "9     service agreement timatinas biopharma holdings   \n",
       "\n",
       "                                        cleaned_name  \n",
       "0  series at chipstead sailing clubamerican civil...  \n",
       "1  serious collision investigation teamap member ...  \n",
       "2    serious crime divisioncamp verde marshal office  \n",
       "3  serious crime divisiondepartment of homeland s...  \n",
       "4  serious crime divisioninternational refugee as...  \n",
       "5                              serpay folklore group  \n",
       "6      serta motion essentials adjustable foundation  \n",
       "7                    service above self registration  \n",
       "8  service agent kerry ogrady facing discipline d...  \n",
       "9  service agreement timatinas biopharma holdings...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## File descriptions\n",
    "\n",
    "Because we need to use pandas, since we have memory issue, we cannot use the whole dataframe, that is why we have 141 dataframes and each dataframe contains 500k rows.\n",
    "\n",
    "I will run a example only take 100 rows."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# we take 100 to 199 rows as a sample dataframe\n",
    "sample_df = new_df.loc[100:199]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function that help to decision if the result is an org or not\n",
    "def getDecision(input_list):\n",
    "    for item in input_list:\n",
    "        if item[1] == 'ORGANIZATION':\n",
    "            return \"Yes\"\n",
    "            break\n",
    "        \n",
    "    return \"No\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ner_result(input_str):\n",
    "    new_str = ' '.join([w.capitalize() for w in input_str.split(' ')])\n",
    "    tokenized_text = word_tokenize(new_str)\n",
    "    classified_text = st.tag(tokenized_text)\n",
    "    return getDecision(classified_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "# Here I'm using m5.12xlarge virtual machine, and I'll use datetime to show you the estimate running time.\n",
    "# You can also add several rows in your code to give a count down."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162.047487\n"
     ]
    }
   ],
   "source": [
    "result_df = sample_df.copy()\n",
    "initial_time = datetime.datetime.now()\n",
    "result_df['Decision'] = result_df['cleaned_name'].apply(ner_result)\n",
    "print((datetime.datetime.now() - initial_time).total_seconds())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_name</th>\n",
       "      <th>cleaned_name</th>\n",
       "      <th>Decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>services indicestwitter</td>\n",
       "      <td>services indicestwitter</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>services league ivan venningfinancial modellin...</td>\n",
       "      <td>services league ivan venningfinancial modellin...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>services league ivan venninggeneva refugee con...</td>\n",
       "      <td>services league ivan venninggeneva refugee con...</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>services league ivan venningomnia ibrahim egyp...</td>\n",
       "      <td>services league ivan venningomnia ibrahim egyp...</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>services login</td>\n",
       "      <td>services login</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>services mainbehavior blocker</td>\n",
       "      <td>services mainbehavior blocker</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>services maincentral intelligence agency</td>\n",
       "      <td>services maincentral intelligence agency</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>services maindistrict reserve groupabu dhabi n...</td>\n",
       "      <td>services maindistrict reserve groupabu dhabi n...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>services mainsociety toreuters</td>\n",
       "      <td>services mainsociety toreuters</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>services mainunited statesqueen collegedepartm...</td>\n",
       "      <td>services mainunited statesqueen collegedepartm...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         original_name  \\\n",
       "100                            services indicestwitter   \n",
       "101  services league ivan venningfinancial modellin...   \n",
       "102  services league ivan venninggeneva refugee con...   \n",
       "103  services league ivan venningomnia ibrahim egyp...   \n",
       "104                                     services login   \n",
       "105                      services mainbehavior blocker   \n",
       "106           services maincentral intelligence agency   \n",
       "107  services maindistrict reserve groupabu dhabi n...   \n",
       "108                     services mainsociety toreuters   \n",
       "109  services mainunited statesqueen collegedepartm...   \n",
       "\n",
       "                                          cleaned_name Decision  \n",
       "100                            services indicestwitter       No  \n",
       "101  services league ivan venningfinancial modellin...      Yes  \n",
       "102  services league ivan venninggeneva refugee con...       No  \n",
       "103  services league ivan venningomnia ibrahim egyp...       No  \n",
       "104                                     services login       No  \n",
       "105                      services mainbehavior blocker       No  \n",
       "106           services maincentral intelligence agency      Yes  \n",
       "107  services maindistrict reserve groupabu dhabi n...      Yes  \n",
       "108                     services mainsociety toreuters       No  \n",
       "109  services mainunited statesqueen collegedepartm...      Yes  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output_df = result_df[result_df.Decision == 'Yes']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_name</th>\n",
       "      <th>cleaned_name</th>\n",
       "      <th>Decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>services league ivan venningfinancial modellin...</td>\n",
       "      <td>services league ivan venningfinancial modellin...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>services maincentral intelligence agency</td>\n",
       "      <td>services maincentral intelligence agency</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>services maindistrict reserve groupabu dhabi n...</td>\n",
       "      <td>services maindistrict reserve groupabu dhabi n...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>services mainunited statesqueen collegedepartm...</td>\n",
       "      <td>services mainunited statesqueen collegedepartm...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>services public relationsdisa courtschmidt nat...</td>\n",
       "      <td>services public relationsdisa courtschmidt nat...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>services public relationsmayo performing arts ...</td>\n",
       "      <td>services public relationsmayo performing arts ...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>services taxforeign ministry for american affa...</td>\n",
       "      <td>services taxforeign ministry for american affa...</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>services taxjames walker group</td>\n",
       "      <td>services taxjames walker group</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>services taxstag companyassetmark</td>\n",
       "      <td>services taxstag companyassetmark inc</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>services viagraafghan for strategic</td>\n",
       "      <td>services viagraafghan institute for strategic</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         original_name  \\\n",
       "101  services league ivan venningfinancial modellin...   \n",
       "106           services maincentral intelligence agency   \n",
       "107  services maindistrict reserve groupabu dhabi n...   \n",
       "109  services mainunited statesqueen collegedepartm...   \n",
       "111  services public relationsdisa courtschmidt nat...   \n",
       "112  services public relationsmayo performing arts ...   \n",
       "120  services taxforeign ministry for american affa...   \n",
       "124                     services taxjames walker group   \n",
       "127                  services taxstag companyassetmark   \n",
       "133                services viagraafghan for strategic   \n",
       "\n",
       "                                          cleaned_name Decision  \n",
       "101  services league ivan venningfinancial modellin...      Yes  \n",
       "106           services maincentral intelligence agency      Yes  \n",
       "107  services maindistrict reserve groupabu dhabi n...      Yes  \n",
       "109  services mainunited statesqueen collegedepartm...      Yes  \n",
       "111  services public relationsdisa courtschmidt nat...      Yes  \n",
       "112  services public relationsmayo performing arts ...      Yes  \n",
       "120  services taxforeign ministry for american affa...      Yes  \n",
       "124                     services taxjames walker group      Yes  \n",
       "127              services taxstag companyassetmark inc      Yes  \n",
       "133      services viagraafghan institute for strategic      Yes  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Only output the first two column which is original_name and cleaned_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output_df = output_df[['original_name','cleaned_name']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_name</th>\n",
       "      <th>cleaned_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>services league ivan venningfinancial modellin...</td>\n",
       "      <td>services league ivan venningfinancial modellin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>services maincentral intelligence agency</td>\n",
       "      <td>services maincentral intelligence agency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>services maindistrict reserve groupabu dhabi n...</td>\n",
       "      <td>services maindistrict reserve groupabu dhabi n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>services mainunited statesqueen collegedepartm...</td>\n",
       "      <td>services mainunited statesqueen collegedepartm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>services public relationsdisa courtschmidt nat...</td>\n",
       "      <td>services public relationsdisa courtschmidt nat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>services public relationsmayo performing arts ...</td>\n",
       "      <td>services public relationsmayo performing arts ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>services taxforeign ministry for american affa...</td>\n",
       "      <td>services taxforeign ministry for american affa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>services taxjames walker group</td>\n",
       "      <td>services taxjames walker group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>services taxstag companyassetmark</td>\n",
       "      <td>services taxstag companyassetmark inc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>services viagraafghan for strategic</td>\n",
       "      <td>services viagraafghan institute for strategic</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         original_name  \\\n",
       "101  services league ivan venningfinancial modellin...   \n",
       "106           services maincentral intelligence agency   \n",
       "107  services maindistrict reserve groupabu dhabi n...   \n",
       "109  services mainunited statesqueen collegedepartm...   \n",
       "111  services public relationsdisa courtschmidt nat...   \n",
       "112  services public relationsmayo performing arts ...   \n",
       "120  services taxforeign ministry for american affa...   \n",
       "124                     services taxjames walker group   \n",
       "127                  services taxstag companyassetmark   \n",
       "133                services viagraafghan for strategic   \n",
       "\n",
       "                                          cleaned_name  \n",
       "101  services league ivan venningfinancial modellin...  \n",
       "106           services maincentral intelligence agency  \n",
       "107  services maindistrict reserve groupabu dhabi n...  \n",
       "109  services mainunited statesqueen collegedepartm...  \n",
       "111  services public relationsdisa courtschmidt nat...  \n",
       "112  services public relationsmayo performing arts ...  \n",
       "120  services taxforeign ministry for american affa...  \n",
       "124                     services taxjames walker group  \n",
       "127              services taxstag companyassetmark inc  \n",
       "133      services viagraafghan institute for strategic  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_df.head(10)\n",
    "# REMEMBER TO SAVE FILE"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}