242Proj_final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7pBiX9qi30yf"
   },
   "outputs": [],
   "source": [
    "# Import basic libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import os\n",
    "import string\n",
    "from dateutil import parser\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "%matplotlib inline\n",
    "\n",
    "data_dir=\"./\"\n",
    "profiles_path = os.path.join(data_dir, 'linkedin.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P_4GjxRM30yq",
    "outputId": "2d34195d-e2af-43c6-ffea-aa1383aafa77"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of profiles loaded from JSON: 1000000\n"
     ]
    }
   ],
   "source": [
    "# Read the data from the JSON file https://drive.google.com/open?id=1fXcY3pQc32sqdaOPft9EeKFMpqbYNTbN\n",
    "# reading the first 1,000,000 profile\n",
    "num_profiles_from_JSON = 1000000\n",
    "profiles = []\n",
    "\n",
    "for line in open(profiles_path):\n",
    "    if len(profiles) < num_profiles_from_JSON:\n",
    "        line = json.loads(line)\n",
    "        profiles.append(line)\n",
    "        \n",
    "    else:\n",
    "        break\n",
    "\n",
    "print('Number of profiles loaded from JSON:', len(profiles))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2jJohnIe30yo"
   },
   "outputs": [],
   "source": [
    "# Function that remove punctuation from strings\n",
    "def remove_punctuation(s):\n",
    "    translator = str.maketrans('', '', string.punctuation) \n",
    "    return s.translate(translator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fm3QKGbN30yv",
    "outputId": "ea6b6343-2aae-41bd-8fa3-2fdbf50dd6e3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking language...\n",
      "Number of profiles with experience, education, and skills: 455792\n",
      "Number of profiles removed because it is not in english: 348160\n",
      "Percentage of profiles removed because it is not in english: 76.39%\n",
      "Number of profiles loaded into the dataFrame: 107632\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.corpus import wordnet\n",
    "import random\n",
    "random.seed(2)\n",
    "\n",
    "# Extract features from list of dictionaries and put into a dataframe\n",
    "feature_extraction_list = ['skills','specilities','interests','experience','education','summary','honors','industry','group']\n",
    "df = pd.DataFrame(columns=feature_extraction_list)\n",
    "rows_list = []\n",
    "\n",
    "# Parameters for checking English\n",
    "number_of_word_to_test = 20 # Number of random words picked from 'experience' to check for English\n",
    "non_eng_threshold = 0.25 # Percentage of non-english words allowed during checking before removing a profile\n",
    "\n",
    "print('Checking language...')\n",
    "num_profiles_with_features = 0\n",
    "num_profiles_lang_not_en = 0\n",
    "for i in profiles:\n",
    "    #only select profiles with data for experience, education, and skills\n",
    "    if (\"experience\" in i) and (\"education\" in i) and (\"skills\" in i):\n",
    "        num_profiles_with_features = num_profiles_with_features + 1\n",
    "\n",
    "        #check if 'industry' string is in English\n",
    "        if 'industry' in i:\n",
    "            industry_is_english = True\n",
    "            word_list = remove_punctuation(str(i['industry'])).lower().split()\n",
    "            for k in word_list:\n",
    "                if not wordnet.synsets(k):\n",
    "                    industry_is_english = False\n",
    "        else:\n",
    "            industry_is_english = True\n",
    "\n",
    "        #check if there are enough words in 'experience' that are in English\n",
    "        if industry_is_english == True:\n",
    "            word_list = remove_punctuation(str(i['experience'])).lower().split()\n",
    "            word_list_rand = []\n",
    "            for k in range(number_of_word_to_test):\n",
    "                word_list_rand.append(word_list[random.randrange(1, len(word_list))])\n",
    "            eng_count = 0\n",
    "            for k in word_list_rand:\n",
    "                if wordnet.synsets(k):\n",
    "                    eng_count = eng_count + 1           \n",
    "            if eng_count/len(word_list_rand) >= (1 - non_eng_threshold):\n",
    "                is_english = True\n",
    "            else:\n",
    "                is_english = False\n",
    "        else: \n",
    "            is_english = False\n",
    "\n",
    "        #if the profile is qualified to be in English, extract the features to put in a dataframe    \n",
    "        if is_english == True:            \n",
    "            dict1 = {}\n",
    "            for j in feature_extraction_list:\n",
    "                if j in i:\n",
    "                    dict1[j] = i[j]\n",
    "                else:\n",
    "                    if j in ['specilities','interests','summary','industry']:\n",
    "                        dict1[j] = ''\n",
    "                    elif j in ['skills','honors','group']:\n",
    "                        dict1[j] = []\n",
    "                    else:\n",
    "                        dict1[j] = float('nan')            \n",
    "            rows_list.append(dict1)\n",
    "        else:\n",
    "            num_profiles_lang_not_en = num_profiles_lang_not_en + 1\n",
    "\n",
    "df = pd.DataFrame(rows_list)\n",
    "print('Number of profiles with experience, education, and skills:', num_profiles_with_features)\n",
    "print('Number of profiles removed because it is not in english:', num_profiles_lang_not_en)\n",
    "print('Percentage of profiles removed because it is not in english: ', round(num_profiles_lang_not_en/num_profiles_with_features*100, 2),'%', sep='')\n",
    "print('Number of profiles loaded into the dataFrame:', len(rows_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ngc75BBU30yy"
   },
   "outputs": [],
   "source": [
    "#split data from education and experience into the following columns\n",
    "df['occupation'] = ''\n",
    "df['company'] = ''\n",
    "df['major'] = ''\n",
    "df['institution'] = ''\n",
    "df['degree'] = ''\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    if type(row[\"experience\"]) == list:\n",
    "        temp_occupation = []\n",
    "        temp_company = []\n",
    "        for i in row['experience']:\n",
    "            if 'title' in i:\n",
    "                temp_occupation.append(i['title'])\n",
    "            else:\n",
    "                temp_occupation.append('')\n",
    "            if 'org' in i:\n",
    "                temp_company.append(i['org'])\n",
    "            else:\n",
    "                temp_company.append('')        \n",
    "        if len(temp_occupation) == 0:\n",
    "            row['occupation'] = []\n",
    "        else:\n",
    "            row['occupation'] = temp_occupation\n",
    "        if len(temp_company) == 0:\n",
    "            row['company'] = []\n",
    "        else:\n",
    "            row['company'] = temp_company\n",
    "    \n",
    "    if type(row[\"education\"]) == list:\n",
    "        temp_institution = []\n",
    "        temp_major = []\n",
    "        temp_degree = []\n",
    "        for i in row['education']:\n",
    "            if 'name' in i:\n",
    "                temp_institution.append(i['name'])\n",
    "            else:\n",
    "                temp_institution.append('')\n",
    "            if 'major' in i:\n",
    "                temp_major.append(i['major'])\n",
    "            else:\n",
    "                temp_major.append('')              \n",
    "            if 'degree' in i:\n",
    "                temp_degree.append(i['degree'])\n",
    "            else:\n",
    "                temp_degree.append('') \n",
    "                \n",
    "        if len(temp_institution) == 0:\n",
    "            row['institution'] = []\n",
    "        else:\n",
    "            row['institution'] = temp_institution\n",
    "        if len(temp_major) == 0:\n",
    "            row['major'] = []\n",
    "        else:\n",
    "            row['major'] = temp_major\n",
    "        if len(temp_degree) == 0:\n",
    "            row['degree'] = []\n",
    "        else:\n",
    "            row['degree'] = temp_degree            \n",
    "                \n",
    "    if (type(row['group']) == dict and 'affiliation' in row['group']):\n",
    "        row['group'] = row['group']['affiliation']\n",
    "    else:\n",
    "        row['group'] = []\n",
    "        \n",
    "    if row['occupation'] == '':\n",
    "        row['occupation'] = []\n",
    "    if row['company'] == '':\n",
    "        row['company'] = []\n",
    "    if row['major'] == '':\n",
    "        row['major'] = []\n",
    "    if row['institution'] == '':\n",
    "        row['institution'] = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "I0ByPps730y0"
   },
   "outputs": [],
   "source": [
    "# Manually standardize degrees\n",
    "for index, row in df.iterrows():\n",
    "    if type(row['degree']) == list:\n",
    "        temp_degree = []\n",
    "        for degree in row['degree']:\n",
    "            degree = remove_punctuation(degree.upper())\n",
    "\n",
    "            if 'PHD' in degree:\n",
    "                degree = 'PHD'\n",
    "            if \"POST GRADUATE\" in degree:\n",
    "                degree = \"POST GRADUATE\"\n",
    "            if \"MASTER OF SCIENCE\" in degree or 'MSC' in degree or 'MS' in degree:\n",
    "                degree = 'MS'\n",
    "            if \"MASTERS\" in degree or \"MASTER\" in degree:\n",
    "                degree = 'MASTER'\n",
    "            if 'MBA' in degree:\n",
    "                degree = 'MBA'\n",
    "            if \"BACHELOR OF SCIENCE\" in degree or \"BSC\" in degree or \"BS\" in degree:\n",
    "                degree = 'BS'\n",
    "            if \"BACHELOR OF ARTS\" in degree or \"BA\" in degree:\n",
    "                degree = 'BA'\n",
    "            if \"BACHELORS\" in degree or \"BACHELOR\" in degree:\n",
    "                degree = 'BACHELOR'\n",
    "            if 'ASSOCIATE' in degree:\n",
    "                degree = 'ASSOCIATE'\n",
    "            if 'SCIENCE' in degree:\n",
    "                degree = 'SCIENCE'\n",
    "            if 'ENGINEERING' in degree or 'ENGINEER' in degree:\n",
    "                degree = 'ENGINEERING'\n",
    "            if 'BUSINESS' in degree:\n",
    "                degree = 'BUSINESS'\n",
    "            if \"MANAGEMENT\" in degree:\n",
    "                degree = \"MANAGEMENT\"\n",
    "            if 'RESEARCH' in degree:\n",
    "                degree = 'RESEARCH'\n",
    "            if 'HIGH' in degree or 'SECONDARY' in degree:\n",
    "                degree = 'HIGH'\n",
    "            useless_list = ['CERTIFICATE', 'CERTIFICATION', 'CERTIFIED', 'COURSE', 'DE', 'DEGREE', 'DIPLOMA', 'LEVELS', 'LICENCE', 'PROGRAM', 'SCHOOL']\n",
    "            for i in useless_list:\n",
    "                if i in degree:\n",
    "                    degree = \"\"\n",
    "            \n",
    "            temp_degree.append(degree)\n",
    "        if len(temp_degree) == 0:\n",
    "            row['degree'] = []\n",
    "        else:\n",
    "            row['degree'] = temp_degree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BxOzTX6U30y3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of profiles with value for work years: 100.0 %\n"
     ]
    }
   ],
   "source": [
    "# Calculating year of work experience based on the year of data collection and the year of graduation for bachelor degree\n",
    "# Year of entering the workforce is assumed to be the year of graduating with a Bachelor degree or the year of earliest work experience, which one is earlier.\n",
    "year_of_data_collection = 2019\n",
    "\n",
    "df['year_of_work_experience'] = ''\n",
    "for index, row in df.iterrows():\n",
    "    year_min = year_of_data_collection\n",
    "    if type(row['degree']) == list:\n",
    "        for i in range(0,len(row['degree'])):\n",
    "            if row['degree'][i] == 'HIGH':\n",
    "                break\n",
    "            if 'end' in row['education'][i]:\n",
    "                try:\n",
    "                    year_temp = parser.parse(row['education'][i]['end']).year\n",
    "                except:\n",
    "                    pass\n",
    "                    \n",
    "                if year_temp < year_min:\n",
    "                    year_min = year_temp\n",
    "                \n",
    "    if type(row['experience']) == list:\n",
    "        for i in range(0,len(row['experience'])):\n",
    "            if 'start' in row['experience'][i]:\n",
    "                try:\n",
    "                    year_temp = parser.parse(row['experience'][i]['start']).year\n",
    "                except:\n",
    "                    pass\n",
    "                \n",
    "                if year_temp < year_min:\n",
    "                    year_min = year_temp                \n",
    "        \n",
    "    row['year_of_work_experience'] = year_of_data_collection - year_min       \n",
    "\n",
    "    if row['year_of_work_experience'] == '':\n",
    "        row['year_of_work_experience'] = float('nan')\n",
    "        \n",
    "print('Percentage of profiles with value for work years:', \n",
    "      df['year_of_work_experience'].count()/df.shape[0]*100,\n",
    "      '%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rOCUNAVN30y6",
    "outputId": "4e7f7411-75d8-4fad-bf8d-85f1cff6fdab"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>skills</th>\n",
       "      <th>specilities</th>\n",
       "      <th>interests</th>\n",
       "      <th>institution</th>\n",
       "      <th>degree</th>\n",
       "      <th>major</th>\n",
       "      <th>summary</th>\n",
       "      <th>honors</th>\n",
       "      <th>industry</th>\n",
       "      <th>occupation</th>\n",
       "      <th>company</th>\n",
       "      <th>year_of_work_experience</th>\n",
       "      <th>group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[DNA, Nanotechnology, Molecular Biology, Softw...</td>\n",
       "      <td></td>\n",
       "      <td>personal genomics, nanotechnology</td>\n",
       "      <td>[Harvard University, Yale University]</td>\n",
       "      <td>[PHD, BS]</td>\n",
       "      <td>[Biophysics, Computer Science]</td>\n",
       "      <td>I am interested in inventing new methods to co...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Research</td>\n",
       "      <td>[Assistant Professor, Technology Development F...</td>\n",
       "      <td>[UCSF, Wyss Institute for Biologically Inspire...</td>\n",
       "      <td>16</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[Interactive Marketing, Content Strategy, Affi...</td>\n",
       "      <td>Internet Marketing, Interactive Marketing, Dig...</td>\n",
       "      <td>Digital Photography, Tennis, Internet Marketin...</td>\n",
       "      <td>[University of Virginia]</td>\n",
       "      <td>[BA]</td>\n",
       "      <td>[History]</td>\n",
       "      <td>Love to work on deadline &amp; data-driven, dynami...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Internet</td>\n",
       "      <td>[Social Media Marketing Manager, Board of Dire...</td>\n",
       "      <td>[Coca-Cola, Atlanta Interactive Marketing Asso...</td>\n",
       "      <td>21</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[Primavera, Revit MEP, AutoCAD, Engineering, H...</td>\n",
       "      <td></td>\n",
       "      <td>Project Management, Innovative Designs, New Te...</td>\n",
       "      <td>[University of Petroleum &amp; Energy Studies, IIT...</td>\n",
       "      <td>[, , , , ]</td>\n",
       "      <td>[, , , , ]</td>\n",
       "      <td>More than 08 yrs of experience with hands on e...</td>\n",
       "      <td>[# Selected in ARMY through SSB (Technical Gra...</td>\n",
       "      <td>Oil &amp; Energy</td>\n",
       "      <td>[Manager (International Business Development &amp;...</td>\n",
       "      <td>[VOITH Hydro Pvt Ltd., VOITH Hydro Pvt. Ltd., ...</td>\n",
       "      <td>22</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Talent Acquisition, Recruiting, Talent Manage...</td>\n",
       "      <td></td>\n",
       "      <td>Organizational development, personal developme...</td>\n",
       "      <td>[Universitatea „Transilvania” din Brașov, Univ...</td>\n",
       "      <td>[BA, ]</td>\n",
       "      <td>[Psychology &amp; Science of Education, ]</td>\n",
       "      <td>RecruitmentInternal communicationTrainingNew h...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Human Resources</td>\n",
       "      <td>[Recruitment consultant, Scientific Staffing C...</td>\n",
       "      <td>[CGI, Kelly Services, Carmeuse, Education Inst...</td>\n",
       "      <td>18</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[Brand Management, Integrated Marketing, Telev...</td>\n",
       "      <td></td>\n",
       "      <td>Broadcast Design, Television, Creativity, Music.</td>\n",
       "      <td>[Instituto de Diseño de Caracas]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>Accomplished Chief Creative Officer with 15+ y...</td>\n",
       "      <td>[Promax and BDA World Gold Awards, New York Fe...</td>\n",
       "      <td>Broadcast Media</td>\n",
       "      <td>[Creative Services, Vice President, Creative S...</td>\n",
       "      <td>[Discovery Communications, Warner Channel, War...</td>\n",
       "      <td>28</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              skills  \\\n",
       "0  [DNA, Nanotechnology, Molecular Biology, Softw...   \n",
       "1  [Interactive Marketing, Content Strategy, Affi...   \n",
       "2  [Primavera, Revit MEP, AutoCAD, Engineering, H...   \n",
       "3  [Talent Acquisition, Recruiting, Talent Manage...   \n",
       "4  [Brand Management, Integrated Marketing, Telev...   \n",
       "\n",
       "                                         specilities  \\\n",
       "0                                                      \n",
       "1  Internet Marketing, Interactive Marketing, Dig...   \n",
       "2                                                      \n",
       "3                                                      \n",
       "4                                                      \n",
       "\n",
       "                                           interests  \\\n",
       "0                  personal genomics, nanotechnology   \n",
       "1  Digital Photography, Tennis, Internet Marketin...   \n",
       "2  Project Management, Innovative Designs, New Te...   \n",
       "3  Organizational development, personal developme...   \n",
       "4   Broadcast Design, Television, Creativity, Music.   \n",
       "\n",
       "                                         institution      degree  \\\n",
       "0              [Harvard University, Yale University]   [PHD, BS]   \n",
       "1                           [University of Virginia]        [BA]   \n",
       "2  [University of Petroleum & Energy Studies, IIT...  [, , , , ]   \n",
       "3  [Universitatea „Transilvania” din Brașov, Univ...      [BA, ]   \n",
       "4                   [Instituto de Diseño de Caracas]          []   \n",
       "\n",
       "                                   major  \\\n",
       "0         [Biophysics, Computer Science]   \n",
       "1                              [History]   \n",
       "2                             [, , , , ]   \n",
       "3  [Psychology & Science of Education, ]   \n",
       "4                                     []   \n",
       "\n",
       "                                             summary  \\\n",
       "0  I am interested in inventing new methods to co...   \n",
       "1  Love to work on deadline & data-driven, dynami...   \n",
       "2  More than 08 yrs of experience with hands on e...   \n",
       "3  RecruitmentInternal communicationTrainingNew h...   \n",
       "4  Accomplished Chief Creative Officer with 15+ y...   \n",
       "\n",
       "                                              honors         industry  \\\n",
       "0                                                 []         Research   \n",
       "1                                                 []         Internet   \n",
       "2  [# Selected in ARMY through SSB (Technical Gra...     Oil & Energy   \n",
       "3                                                 []  Human Resources   \n",
       "4  [Promax and BDA World Gold Awards, New York Fe...  Broadcast Media   \n",
       "\n",
       "                                          occupation  \\\n",
       "0  [Assistant Professor, Technology Development F...   \n",
       "1  [Social Media Marketing Manager, Board of Dire...   \n",
       "2  [Manager (International Business Development &...   \n",
       "3  [Recruitment consultant, Scientific Staffing C...   \n",
       "4  [Creative Services, Vice President, Creative S...   \n",
       "\n",
       "                                             company year_of_work_experience  \\\n",
       "0  [UCSF, Wyss Institute for Biologically Inspire...                      16   \n",
       "1  [Coca-Cola, Atlanta Interactive Marketing Asso...                      21   \n",
       "2  [VOITH Hydro Pvt Ltd., VOITH Hydro Pvt. Ltd., ...                      22   \n",
       "3  [CGI, Kelly Services, Carmeuse, Education Inst...                      18   \n",
       "4  [Discovery Communications, Warner Channel, War...                      28   \n",
       "\n",
       "  group  \n",
       "0    []  \n",
       "1    []  \n",
       "2    []  \n",
       "3    []  \n",
       "4    []  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#clean up the dataframe\n",
    "df = df[['skills','specilities','interests',\n",
    "               'institution','degree','major',\n",
    "               'summary','honors','industry',\n",
    "               'occupation','company',\n",
    "               'year_of_work_experience','group']].copy()\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Word2Vec and nltk libraries\n",
    "import re\n",
    "from gensim.models import KeyedVectors\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models.phrases import Phraser, Phrases\n",
    "import unicodedata\n",
    "from nltk import word_tokenize, sent_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import LancasterStemmer, WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function that removes English stopwords\n",
    "def remove_stopwords(words):\n",
    "    stopword_list = stopwords.words('english')\n",
    "    for word in words:\n",
    "        if word in stopword_list:\n",
    "            words.remove(word)\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean, lemmatize and tokenize the \"skills\" column for all profiles\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "skills = list()\n",
    "for index,row in df.iterrows():\n",
    "    r = list()\n",
    "    for i in row['skills']:\n",
    "        i=i.translate(str.maketrans('', '', string.punctuation))\n",
    "        i=i.lower()\n",
    "        i=lemmatizer.lemmatize(i,pos='n')\n",
    "        i=nltk.word_tokenize(i)\n",
    "        i=remove_stopwords(i)\n",
    "        r.append(i)\n",
    "    skills.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use bigram to create Word2Vec model for \"skills\"\n",
    "all_sentences2 = []\n",
    "for text in skills:\n",
    "    all_sentences2 += text\n",
    "    \n",
    "phrases = Phrases(all_sentences2, min_count=1, threshold=1)\n",
    "bigram=Phraser(phrases)\n",
    "all_sentences2 = list(bigram[all_sentences2])\n",
    "\n",
    "model2 = Word2Vec(all_sentences2, \n",
    "                 min_count=100,   # Ignore words that appear less than this\n",
    "                 size=200,      # Dimensionality of word embeddings\n",
    "                 workers=16,     # Number of processors (parallelisation)\n",
    "                 window=5,      # Context window for words during training\n",
    "                 iter=30)       # Number of epochs training over corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean, lemmatize and tokenize the \"major\" column for all profiles\n",
    "major = list()\n",
    "for index,row in df.iterrows():\n",
    "    r = list()\n",
    "    for i in row['major']:\n",
    "        i=i.translate(str.maketrans('', '', string.punctuation))\n",
    "        i=i.lower()\n",
    "        i=lemmatizer.lemmatize(i,pos='n')\n",
    "        i=nltk.word_tokenize(i)\n",
    "        i=remove_stopwords(i)\n",
    "        r.append(i)\n",
    "    major.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use bigram to create Word2Vec model for \"major\"\n",
    "all_sentences3 = []\n",
    "for text in major:\n",
    "    all_sentences3 += text\n",
    "    \n",
    "phrases = Phrases(all_sentences3, min_count=1,threshold=1)\n",
    "bigram=Phraser(phrases)\n",
    "all_sentences3 = list(bigram[all_sentences3])\n",
    "\n",
    "model3 = Word2Vec(all_sentences3, \n",
    "                 min_count=200,   # Ignore words that appear less than this\n",
    "                 size=100,      # Dimensionality of word embeddings\n",
    "                 workers=16,     # Number of processors (parallelisation)\n",
    "                 window=5,      # Context window for words during training\n",
    "                 iter=30)       # Number of epochs training over corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean and tokenize the \"degree\" column for all profiles\n",
    "degree = list()\n",
    "for index,row in df.iterrows():\n",
    "    r = list()\n",
    "    for i in row['degree']:\n",
    "        i=i.translate(str.maketrans('', '', string.punctuation))\n",
    "        i=i.lower()\n",
    "        i=nltk.word_tokenize(i)\n",
    "        i=remove_stopwords(i)\n",
    "        r.append(i)\n",
    "    degree.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use bigram to create Word2Vec model for \"degree\"\n",
    "all_sentences4 = []\n",
    "for text in degree:\n",
    "    all_sentences4 += text\n",
    "    \n",
    "phrases = Phrases(all_sentences4, min_count=1,threshold=1)\n",
    "bigram=Phraser(phrases)\n",
    "all_sentences4 = list(bigram[all_sentences4])\n",
    "\n",
    "model4 = Word2Vec(all_sentences4, \n",
    "                 min_count=200,   # Ignore words that appear less than this\n",
    "                 size=50,      # Dimensionality of word embeddings\n",
    "                 workers=16,     # Number of processors (parallelisation)\n",
    "                 window=5,      # Context window for words during training\n",
    "                 iter=30)       # Number of epochs training over corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean and lemmatize the \"institution\" column for all profiles\n",
    "institution = list()\n",
    "for index,row in df.iterrows():\n",
    "    r = list()\n",
    "    for i in row['institution']:\n",
    "        i=i.translate(str.maketrans('', '', string.punctuation))\n",
    "        i=i.lower()\n",
    "        i=lemmatizer.lemmatize(i,pos='n')\n",
    "        r.append(i)\n",
    "    institution.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use bigram to create Word2Vec model for \"institution\"\n",
    "all_sentences5 = []\n",
    "for text in institution:\n",
    "    all_sentences5.append(list(text))\n",
    "    \n",
    "phrases = Phrases(all_sentences5, min_count=1,threshold=1)\n",
    "bigram=Phraser(phrases)\n",
    "all_sentences5 = list(bigram[all_sentences5])\n",
    "\n",
    "model5 = Word2Vec(all_sentences5, \n",
    "                 min_count=20,   # Ignore words that appear less than this\n",
    "                 size=200,      # Dimensionality of word embeddings\n",
    "                 workers=16,     # Number of processors (parallelisation)\n",
    "                 window=5,      # Context window for words during training\n",
    "                 iter=30)       # Number of epochs training over corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean, lemmatize and tokenize the \"industry\" column for all profiles\n",
    "industry = list()\n",
    "for i in df['industry']:\n",
    "    r = list()\n",
    "    if type(i) == float:\n",
    "        i = \"\"\n",
    "    i=i.translate(str.maketrans('', '', string.punctuation))\n",
    "    i=i.lower()\n",
    "    i=lemmatizer.lemmatize(i,pos='n')\n",
    "    i=nltk.word_tokenize(i)\n",
    "    i=remove_stopwords(i)\n",
    "    r.append(i)\n",
    "    industry.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use bigram to create Word2Vec model for \"industry\"\n",
    "all_sentences6 = []\n",
    "for text in industry:\n",
    "    all_sentences6 += text\n",
    "    \n",
    "phrases = Phrases(all_sentences6, min_count=1,threshold=1)\n",
    "bigram=Phraser(phrases)\n",
    "all_sentences6 = list(bigram[all_sentences6])\n",
    "\n",
    "model6 = Word2Vec(all_sentences6, \n",
    "                 min_count=50,   # Ignore words that appear less than this\n",
    "                 size=200,      # Dimensionality of word embeddings\n",
    "                 workers=16,     # Number of processors (parallelisation)\n",
    "                 window=5,      # Context window for words during training\n",
    "                 iter=30)       # Number of epochs training over corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean and lemmatize the \"company\" column for all profiles\n",
    "company = list()\n",
    "for index,row in df.iterrows():\n",
    "    r = list()\n",
    "    for i in row['company']:\n",
    "        i=i.translate(str.maketrans('', '', string.punctuation))\n",
    "        i=i.lower()\n",
    "        i=lemmatizer.lemmatize(i,pos='n')\n",
    "        r.append(i)\n",
    "    company.append(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use bigram to create Word2Vec model for \"company\"\n",
    "all_sentences7 = []\n",
    "for text in company:\n",
    "    all_sentences7.append(list(text))\n",
    "    \n",
    "phrases = Phrases(all_sentences7, min_count=1,threshold=1)\n",
    "bigram=Phraser(phrases)\n",
    "all_sentences7 = list(bigram[all_sentences7])\n",
    "\n",
    "model7 = Word2Vec(all_sentences7, \n",
    "                 min_count=20,   # Ignore words that appear less than this\n",
    "                 size=200,      # Dimensionality of word embeddings\n",
    "                 workers=16,     # Number of processors (parallelisation)\n",
    "                 window=5,      # Context window for words during training\n",
    "                 iter=30)       # Number of epochs training over corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1949\n",
      "152\n",
      "38\n",
      "1456\n",
      "146\n",
      "1354\n"
     ]
    }
   ],
   "source": [
    "print(len(model2.wv.vocab)) # skill\n",
    "print(len(model3.wv.vocab)) # major\n",
    "print(len(model4.wv.vocab)) # degree\n",
    "print(len(model5.wv.vocab)) # institution\n",
    "print(len(model6.wv.vocab)) # industry\n",
    "print(len(model7.wv.vocab)) # company"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hierarchical clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of clusters (tunable)\n",
    "n_cluster_2 = 50\n",
    "n_cluster_3 = 20\n",
    "n_cluster_5 = 50\n",
    "n_cluster_6 = 10\n",
    "n_cluster_7 = 50\n",
    "n_cluster_label = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "['mobile', 'network', 'architecture', 'object_oriented', 'web', 'data', 'database', 'user_interface', 'security', 'user_experience', 'web_application', 'solution', 'hardware']\n",
      "\n",
      "\n",
      "['content', 'sales', 'brand', 'product', 'key_account', 'account', 'community', 'workforce', 'campaign', 'customer', 'field', 'channel', 'client', 'human_capital', 'category', 'materials', 'sponsorship']\n",
      "\n",
      "\n",
      "['healthcare_information', 'home', 'writing', 'trading', 'industry', 'food', 'policy', 'trade', 'consumer', 'research', 'science', 'products', 'high']\n",
      "\n",
      "\n",
      "['nanotechnology', 'molecular_biology', 'interactive_marketing', 'web_analytics', 'power_plants', 'epc', 'project_execution', 'revit', 'power_generation', 'primavera_p6', 'creative_direction', 'cs', 'joomla', 'drupal', 'magento', 'sdlc', 'benefits_administration', 'hr_policies', 'onboarding', 'performance_appraisal', 'fmla', 'succession_planning', 'employee_engagement', 'organizational_development', 'recruitmentretention', 'competitive_analysis', 'blackberry', 'telephony', 'voip', 'unified_communications', 'wan', 'iptv', 'networks', 'brand_awareness', 'thought_leadership', 'fha', 'mortgage_banking', 'clinical_trials', 'cro', 'culture_change', 'fmcg', 'oncology', 'pharmaceutical_industry', 'medical_devices', 'financial_modeling', 'career_counseling', 'web_20', 'acquisition_integration', 'corporate_governance', 'sa', 'accounts_payable', 'vat', 'mortgage_lending', 'refinance', 'project_implementation', 'hospitality_industry', 'vhdl', 'arm', 'uml', 'tcpip', 'embedded_systems', 'embedded_software', 'device_drivers', 'distributed_systems', 'multithreading', 'investment_strategies', 'investment_banking', 'pharmaceutical_sales', 'cardiology', 'neurology', 'managed_care', 'diabetes', 'recruitment_advertising', 'technical_recruiting', 'graduate_recruitment', 'financial_markets', 'youtube', 'video_editing', 'window', 'tcl', 'market_research', 'requirements_gathering', 'stakeholder_engagement', 'entrepreneur', 'people_skills', 'concept_design', 'video_games', 'storyboarding', 'character_animation', 'corel_draw', 'licensing', 'email_marketing', 'lead_generation', 'online_advertising', 'media_buying', 'customer_retention', 'gsm', '3g', 'managed_services', 'strategic_sourcing', 'food_industry', 'wcf', 'tdd', 'aspnet_mvc', 'oop', 'silverlight', 'tsql', 'linq', 'enterprise_architecture', 'scrum', 'tfs', 'wpf', 'ssis', 'relational_databases', 'debugging', 'accounts_receivable', 'capital_equipment', 'diagnostics', 'mba', 'customer_relations', 'back_office', 'arcgis', 'corporate_communications', 'internal_communications', 'latex', 'numerical_analysis', 'fortran', 'mathematica', 'revenue_recognition', 'oracle_applications', 'shopper_marketing', 'consumer_products', 'grocery', 'financial_accounting', 'cpr_certified', 'cpa', 'us_gaap', 'hr_consulting', 'sap_r3', 'middleware', 'gold', 'computer_science', 'signal_processing', 'neuroscience', 'biomedical_engineering', 'machine_learning', 'image_processing', 'product_launch', 'fast_learner', 'venture_capital', 'asset_allocation', 'information_retrieval', 'scalability', 'mapreduce', 'data_mining', 'petroleum_engineering', 'oilgas', 'pump', 'offshore_drilling', 'psychotherapy', 'offshoring', 'community_outreach', 'ansys', 'proposal_writing', 'capacity_building', 'income_statement', 'typography', 'logo_design', 'art_direction', 'corporate_identity', 'digital_photography', 'poster', 'painting', 'vendor_relations', 'demand_generation', 'contract_negotiations', 'interior_design', 'va', 'emerging_markets', 'network_administration', 'ubuntu', 'mobile_devices', 'gotomarket_strategy', 'business_transformation', 'mobile_applications', 'hibernate', 'eclipse', 'servlets', 'tomcat', 'ejb', 'jboss', 'subversion', 'apache', 'ant', 'jdbc', 'junit', 'shell_scripting', 'plsql', 'jms', 'energy_industry', 'legal_advice', 'renewable_energy', 'electrical_engineering', 'performance_tuning', 'adonet', 'information_architecture', 'html5', 'html_5', 'lean_manufacturing', 'army', 'earthwork', 'schedule', 'preconstruction', 'subcontracting', 'network_security', 'vpn', 'survey_design', 'executive_coaching', 'balanced_scorecard', 'direct_sales', 'texturing', 'perforce', 'computer_graphics', '3d_modeling', 'cold_calling', 'litigation_support', 'website_development', 'google_analytics', 'characterization', 'chromatography', 'visual_communication', 'flash_animation', 'architectural_design', 'ruby_rails', 'maya', 'motion_graphics', 'qualitative_research', 'data_collection', 'grant_writing', 'econometrics', 'json', 'git', 'lamp', 'solution_selling', 'open_source', 'technology_transfer', 'disability_insurance', '401k', 'income_protection', 'trading_systems', 'quantitative_research', 'leed_ap', 'project_delivery', 'solidworks', 'continuous_improvement', 'multichannel_marketing', 'ps3', 'printing', 'photo_editing', 'keyword_research', 'collection', 'sales_support', 'data_warehousing', 'casualty', 'mechanical_engineering', 'administrative_support', 'strategic_alliances', 'pmo', 'windows_7', 'adobe_acrobat', 'objectivec', 'actionscript', 'ruby', 'dhtml', 'performance_measurement', 'bloomberg', 'government_contracting', 'crm_software', 'sec_reporting', 'cloud_computing', 'micros', 'revenue_analysis', 'system_testing', 'global_sourcing', 'trend_analysis', 'kaizen', 'quality_auditing', 'sybase', 'retail_banking', 'channel_partners', 'business_alliances', 'meeting_planning', 'juniper', 'firewall', 'switch', 'dhcp', 'enterprise_software', 'solution_architecture', 'robotics', 'sensor', 'spectroscopy', 'mixed_signal', 'fire_safety', 'quarkxpress', 'commercial_lending', 'quality_assurance', 'philanthropy', 'green_building', 'energy_efficiency', 'power_distribution', 'program_evaluation', 'admission', 'web_applications', 'lecturing', 'fmea', 'simulink', 'elearning', 'forensic_accounting', 'process_simulation', 'hazop', 'pid', 'equity_trading', 'test_automation', 'credit_cards', 'trade_finance', 'ebusiness', 'proengineer', 'xp', 'interaction_design', 'gi', 'urban_planning', 'bioinformatics', 'genetics', 'pmp', 'market_access', 'solaris', 'resume_writing', 'basel_ii', 'mutual_funds', 'sales_enablement', 'westlaw', 'statistical_modeling', 'usability_testing', 'firmware', 'hadoop', 'big_data', 'router', 'mpls', 'ethernet', 'labview', 'financial_advisory', 'penetration_testing', 'django', 'game_design', 'process_optimization', 'ii', 'quality_center', 'test_cases', 'vbscript', 'design_patterns', 'postgresql', 'soap', 'coating', 'polymer', 'materials_science', 'data_structures', 'public_sector', 'headhunting', 'data_entry', 'adobe_illustrator', 'industrial_design', 'ideation', 'biochemistry', 'medical_education', 'cancer', 'cell_biology', 'workforce_planning', 'hipaa', 'medicare', 'reservoir_engineering', 'lbo', 'structural_analysis', 'environmental_awareness', 'electronic_trading', 'shared_services', 'predictive_analytics', 'patient_safety', 'vulnerability_assessment', 'commercial_litigation', 'commercialization', 'news_writing', 'fixed_assets', 'golf', 'wireless_networking', 'opengl', 'asic', 'bpo', 'linux_kernel', 'alm', 'database_administration', 'disposition', 'water_quality', 'training_delivery', 'interview_preparation', 'lte', 'solvency_ii', 'avid', 'workshop_facilitation', 'urban_design', 'sketchup', 'unity3d', 'jira', '3', 'music_production', 'audio_editing', 'client_relationship', 'civil_engineering', 'structural_engineering', 'sap_erp', 'technical_documentation', 'rhino', 'concept_development', 'model_making', 'ehr', 'rfp', 'cell_culture', 'minitab', 'etl', 'visual_merchandising', 'competitive_intelligence', 'microstation', 'accessibility', 'physical_security', 'mathematics', 'feasibility_studies', 'land_acquisition', 'tax_compliance', 'press_releases', 'ethnography', 'consumer_behaviour', 'university_teaching', 'idea_generation', 'aml', 'maven', 'variance_analysis', 'financial_statements', 'prototyping', 'operating_room', 'microscopy', 'google', 'failure_analysis', 'rehabilitation', 'project_coordination', 'drug_discovery', 'eda', 'apqp', 'catia', 'technical_writing', 'security_clearance', 'power_systems', 'educational_technology', 'preopening', 'flex', 'wimax', 'inside_sales', 'adp_payroll', 'sap_hr', 'lean_sigma', 'food_science', 'national_security', 'soc', 'iso_14001', 'iso_9000', 'visual_effects', 'lanwan', 'qos', 'bash', 'situational_leadership', 'published_author', 'bookkeeping', 'referral', 'pattern_recognition', 'online_journalism', '5', 'tax_advisory', 'capital_budgeting', 'clinical_research', 'brand_equity', 'short_sales', 'foreclosure', 'dutch', 'proofreading', 'application_architecture', 'pro_tools', 'core_java', 'hydraulics', 'io', 'xslt', 'cmmi', 'clearcase', 'virtual_teams', 'ngo', 'customer_focus', 'microcontrollers', 'lightroom', 'job_evaluation', 'invoicing', 'software_implementation', 'bim', 'copy_editing', 'dmaic', 'surveying', 'oilfield', 'logo', 'creative_writing', 'television_production', 'broadband', 'gcp', 'curriculum_development', 'singing', 'songwriting', 'politics', 'investigation', 'cryptography', 'project_estimation', 'mass_spectrometry', 'refinery', 'capa', 'hplc', 'metric', 'gps', 'tax_returns', 'immunology', 'vlsi', 'bgp', 'ospf', 'data_migration', 'curriculum_design', 'high_availability', 'industrial_safety', 'construction_safety', 'accrual', 'unit_testing', 'tax_preparation', 'ipo', 'post_production', 'broadcast_television', 'private_sector', 'series_63', 'sip', '3d_visualization', 'cooking', 'community_engagement', 'publicity', 'enforcement', 'vray', 'tenant', 'foreign_policy', 'scada', 'plc', 'dc', 'ecology', 'bs', 'goal_oriented', 'geophysics', 'foreign_languages', 'qt', 'web_standards', 'persona', 'lng', 'mbti', 'metallurgy', 'personal_branding', 'prince2', 'compensation_benefit', 'chemical_engineering', 'space_planning', 'food_processing', 'luxury_goods', 'sewing', 'landlord', 'pharmacology', 'jd_edwards', 'portrait', 'patent_prosecution', 'microbiology', 'computer_forensics', 'listing', 'surgery', 'orthopedic', 'cfd', 'ethic', 'product_launches', 'essbase', 'asme', 'inventor', 'earth_science', 'big_4', 'adolescent', 'water_treatment', 'skin_care', 'home_care', 'life_sciences', 'taleo', 'appeal', 'organic_chemistry', 'culinary_skills', 'quantity_surveying', 'counterterrorism', 'creative_solutions', 'injection_molding', 'groundwater', 'criminal_justice', 'steel_structures']\n",
      "\n",
      "\n",
      "['social', 'communications', 'technology', 'media', 'recruitment', 'education', 'services', 'health', 'solutions', 'student']\n",
      "\n",
      "\n",
      "['social_media', 'facebook', 'twitter', 'creative', 'new_business', 'iphone', 'digital_media', 'relations', 'game', 'b2b', 'android', 'online', 'new', 'strategist', 'linkedin', 'youth', 'b2c', 'sales_force']\n",
      "\n",
      "\n",
      "['customer_service', 'crossfunctional_team', 'team_leadership', 'team_building', 'coaching', 'problem_solving', 'training', 'teaching', 'skills', 'mentoring', 'strong', 'collaborative', 'coach', 'presentation_skills', 'motivation', 'effectiveness', 'interpersonal']\n",
      "\n",
      "\n",
      "['software', 'systems', 'application', 'applications', 'system']\n",
      "\n",
      "\n",
      "['payroll', 'cisco', 'salesforcecom', 'sharepoint', 'crm', 'mac', 'ssrs', 'sap', 'erp', 'operating_systems', 'oracle_sql', 'computer_hardware', 'virtualization', 'hp', 'oracle', 'crystal_reports', 'active_directory', 'microsoft_exchange', 'citrix', 'lotus_notes', 'internet', 'enterprise', 'peoplesoft', 'search', 'technologies', 'cloud', 'ccna', 'dns', 'vmware', 'siebel', 'storage', 'social_games', 'library', 'quickbooks', 'multimedia', 'selenium', 'weblogic', 'microsoft_dynamics', 'db2', 'cognos', 'sage', 'composite', 'pc']\n",
      "\n",
      "\n",
      "['engineering', 'hvac', 'piping', 'inspection', 'commissioning', 'broadcast', 'analog', 'quality_control', 'drainage', 'economics', 'integrity', 'fpga', 'character', '3d', 'automotive', 'traffic', 'customer_experience', 'vehicle', 'algorithm', 'drilling', 'completion', 'operation', 'layout', 'packaging', 'line', 'concrete', 'workshop', 'lighting', 'wordpress', 'validation', 'oil_gas', 'spreadsheet', 'saas', 'frontend', 'data_center', 'profit', 'apparel', 'textile', 'conference', 'mep', 'plant', 'usability', 'human_factors', 'subsea', 'reliability', 'detail', 'rf', 'mining', 'catalog', 'bridge', 'equipment', 'business_objects', 'geology', 'electrical', 'steel', 'material', 'assembly', 'mechanical', 'aircraft', 'image', 'drug', 'engineer', 'blended_learning', 'highway', 'engine', 'efficiency', 'laboratory', 'instrumentation', 'physical', 'case', 'furniture', 'plastic', 'jewelry', 'seismic']\n",
      "\n",
      "\n",
      "['sales_presentations', 'orientation', 'manager', 'outreach', 'trainer', 'six_sigma', 'blogging', 'social_responsibility', 'certified', 'professional', 'orientations', 'director', 'hiring', 'consultant', 'green_belt', 'selection', 'expert', 'relocation', 'analyst', 'project_manager', 'certification', 'instructor']\n",
      "\n",
      "\n",
      "['credit', 'due_diligence', 'finance', 'real_estate', 'debt', 'valuation', 'private_equity', 'investment', 'portfolio', 'capital_markets', 'equity', 'fixed_income', 'law', 'fundraising', 'insurance', 'litigation', 'residential', 'lending', 'transactions', 'property', 'financing']\n",
      "\n",
      "\n",
      "['markets', 'key', 'the', 'mental_health', 'counseling', 'work', 'health_insurance', 'long_term', 'care', 'term_life', 'mediation', 'property_casualty', 'computing', 'advocacy', 'affairs', 'accounts', 'family', 'practice', 'group', 'life', 'master', 'private', 'decision_making', 'homes']\n",
      "\n",
      "\n",
      "['troubleshooting', 'modeling', 'testing', 'simulation', 'optimization', 'automation', 'improvement', 'excellence', 'generation', 'managing', 'integration_testing', 'transformation', 'power', 'modelling', 'designer', 'toyota_production']\n",
      "\n",
      "\n",
      "['planning', 'restructuring', 'forecasting', 'consulting', 'selling', 'scheduling', 'corporate', 'turnaround', 'budgeting', 'operational', 'financial_reporting', 'sustainability', 'estimating', 'advisory', 'advice', 'benefits', 'specialist', 'benefit', 'cost_control']\n",
      "\n",
      "\n",
      "['employee_relations', 'labor_relations', 'conflict_resolution', 'entrepreneurship', 'public_speaking', 'special_events', 'tourism', 'consultative_selling', 'illustration', 'english', 'french', 'mandarin', 'newsletter', 'teamwork', 'psychology', 'brochure', 'russian', 'bilingual', 'google_adwords', 'legal_research', 'staff_training', 'german', 'interview', 'first_aid', 'spanish', 'magazine', 'publication', 'business_acumen', 'japanese', 'staff_development', 'arabic', 'translation', 'military', 'job', 'programs', 'computer_literate', 'critical_thinking', 'multilingual', 'nlp', 'acting', 'theatre', 'emr', 'business_case', 'communication_skills', 'football', 'needs_analysis', 'theory', 'beauty', 'ap_style', 'command', 'analytical_skills', 'cv', 'studies', 'attention_detail', 'soft_skills', 'senior', 'creativity', 'technical_support', 'fluent_english', 'book', 'typing', 'emotional_intelligence', 'chinese', 'italian', 'presenter', 'history', 'investor', 'speaker', 'police', 'selling_skills', 'classroom', 'piano', 'wine', 'tutoring']\n",
      "\n",
      "\n",
      "['performance', 'itil', 'budget', 'outsourcing', 'agile', 'capital', 'infrastructure', 'life_cycle', 'bid', 'site', 'proposal', 'deployment', 'tender', 'websphere']\n",
      "\n",
      "\n",
      "['relationship', 'direct', 'relationships', 'interaction', 'high_performance', 'engagement', 'distributed', 'presentations', 'oriented']\n",
      "\n",
      "\n",
      "['unix_shell', 'java_enterprise', 'edition', 'creative_problem', 'solving', 'user_acceptance', 'performance_indicators', 'human_computer', 'new_hire', 'final_cut', '3d_studio', 'max', 'alternative_dispute', 'resolution', 'environmental_impact', 'amazon_web', 'services_aws', 'first_time', 'home_buyers', 'value_stream', 'collaborative_problem', '21_cfr', 'part_11', 'single_family']\n",
      "\n",
      "\n",
      "['digital', 'web_content', 'video', 'events', 'event', 'visual', 'advertising', 'fine_art', 'landing_page', 'search_engine', 'music', 'fashion', 'electronics', 'film', 'print']\n",
      "\n",
      "\n",
      "['server', 'unix', 'windows', 'linux', 'system_administration', 'red_hat']\n",
      "\n",
      "\n",
      "['business_intelligence', 'pricing', 'digital_signal', 'natural_language', 'processing', 'finite_element', 'intelligence', 'segmentation', 'derivatives', 'acquisition', 'quantitative', 'time_series', 'value', 'google_webmaster', 'profitability', 'root_cause', 'mapping', 'hyperion', 'statistical', 'real', 'problem']\n",
      "\n",
      "\n",
      "['leadership', 'talent_acquisition', 'hr', 'strategic_planning', 'negotiation', 'experience', 'human_resources', 'program', 'event_planning', 'coordination', 'employee', 'initiatives', 'growth', 'programme', 'facilitation', 'negotiations', 'delivery', 'lead', 'collaboration', 'organization', 'execution', 'functional', 'vision', 'editorial', 'alignment', '•']\n",
      "\n",
      "\n",
      "['administration', 'startup', 'call_center', 'small_business', 'offshore', 'higher_education', 'general', 'nonprofit', 'website', 'grant', 'lean', 'office', 'land', 'course']\n",
      "\n",
      "\n",
      "['quality', 'cost', 'service', 'safety', 'test', 'maintenance']\n",
      "\n",
      "\n",
      "['autocad', 'tools', 'data_analysis', 'r', 'language', 'matlab', 'cad', 'stata', 'spss', 'sas', 'vba', 'computer', 'user', 'eviews', 'report_writing', 'monte_carlo', 'advanced']\n",
      "\n",
      "\n",
      "['marketing', 'ppc', 'seo', 'recruiting', 'television', 'audio', 'luxury', 'promotion', 'graphic_design', 'producing', 'sourcing', 'voice', 'drawing', 'art', 'graphic', 'photography', 'sem', 'editing', 'radio', 'interviewing', 'retouching', 'merchandising', 'ecommerce', 'styling', 'animation', 'effects', 'copywriting', 'sound', 'sketching', 'journalism', 'entertainment', 'branding', 'screening', 'routing', 'monetization', 'publishing', 'feed', 'loss_prevention', 'documentary', 'brokerage', 'video_production', 'rendering', 'recording', 'videography', 'casino', 'storytelling', 'pcr', 'newspaper', 'creation', 'interactive', 'composition', 'ic']\n",
      "\n",
      "\n",
      "['affiliate_marketing', 'cm', 'legal_compliance', 'cisco_ios', 'network_architecture', 'cisco_technologies', 'loan_origination', 'pharmacovigilance', 'general_insurance', 'statistic', 'residential_mortgages', 'rtos', 'alternative_investments', 'urology', 'contract_recruitment', 'requirements_analysis', 'ip_networking', 'digital_illustration', 'zbrush', 'entity_framework', 'tortoise_svn', 'agile_methodologies', 'netbeans', 'investment_properties', 'operational_excellence', 'crisis_communications', 'mathematical_modeling', 'motivational_speaking', 'financial_audits', 'sec_filings', 'hyperion_enterprise', 'customer_insight', 'external_audit', 'benefits_negotiation', 'asset_managment', 'system_architecture', 'supervisory_skills', 'brand_developement', 'family_therapy', 'group_therapy', 'corporate_branding', 'account_reconciliation', 'mobile_technology', 'cdma', 'strut', 'jsf', 'xhtml', 'product_innovation', 'software_installation', 'managerial_finance', 'change_orders', 'value_engineering', 'fullcycle_recruiting', 'analytical_chemistry', 'rest', '3d_animation', 'facebook_api', 'data_visualization', 'strategic_partnerships', 'political_science', 'medical_imaging', 'iso_13485', 'team_motivator', 'commodity_markets', 'submittals', 'usercentered_design', 'process_scheduler', 'xbox_360', 'online_publishing', 'smo', 'seo_copywriting', 'legal_writing', 'residential_homes', 'commercial_insurance', 'branding_identity', 'pro_engineer', 'intercultural_communication', 'equity_valuation', 'online_branding', 'operating_budgets', 'screening_resumes', 'structured_finance', 'college_recruiting', 'trade_shows', 'computer_vision', 'frontend_development', 'commercial_banking', 'civil_litigation', 'kanban', 'staffing_services', 'job_coaching', 'sustainable_design', 'verilog', 'distance_learning', 'instructional_design', 'internet_banking', 'financial_structuring', 'sports_marketing', 'genomics', 'computational_biology', 'lifesciences', 'scala', 'scientific_writing', 'iso_27001', 'consumer_insights', 'deferred_compensation', 'equity_derivatives', 'brand_communication', 'subsea_engineering', 'gwt', 'rapid_prototyping', 'parallel_computing', 'scientific_computing', 'conversion_optimization', 'omniture', 'technical_presentations', 'cisa', 'pci_dss', 'security_audits', 'artificial_intelligence', 'text_mining', 'mobile_games', 'semantic_web', 'regression_testing', 'manual_testing', 'predictive_modeling', 'mongodb', 'zend_framework', 'css3', 'organizational_effectiveness', 'humancomputer_interaction', 'fashion_illustration', 'live_events', 'production_managment', 'online_gaming', 'environmental_science', 'mathcad', 'physic', 'organizational_behavior', 'franchising', 'xcode', 'consumer_electronics', 'environmental_policy', 'environmental_compliance', 'feature_films', 'actuarial_science', 'aspnet_ajax', 'wireframes', 'comprehensive_planning', 'premiere', 'pl_responsibility', 'music_industry', 'audio_engineering', 'building_relationships', 'internet_recruiting', 'permanent_placement', 'cost_estimating', '3d_rendering', 'ios_development', 'western_blotting', 'corporate_events', 'portrait_photography', 'jpa', 'gambling', 'cissp', 'credit_derivatives', 'quantitative_finance', 'broadcast_journalism', 'radio_broadcasting', 'continuous_integration', 'mobile_internet', 'viral_marketing', 'geological_mapping', 'petroleum_geology', 'organic_search', 'sarbanesoxley_act', 'construction_drawings', 'spc', 'design_experiments', 'commercial_photography', 'protein_purification', 'elisa', 'unigraphics', 'gdt', 'ppap', 'software_documentation', 'military_operations', 'proprietary_trading', 'bloomberg_terminal', 'job_descriptions', 'consumer_lending', 'cross_selling', 'sdh', 'film_production', 'hr_transformation', 'advanced_css', 'embedded_linux', 'heuristic_evaluation', 'student_affairs', 'industrial_engineering', 'administrative_assistants', 'navy', 'macroeconomics', 'organizational_structure', 'structured_products', 'waterfall', 'mobile_payments', 'employer_branding', 'group_presentations', 'mrp', 'pcb_design', 'computer_security', 'teacher_training', 'educational_leadership', 'infectious_diseases', 'multi_tasking', 'political_campaigns', 'accident_investigation', 'legal_assistance', 'link_building', 'refurbishing', 'glp', 'biopharmaceuticals', 'regulatory_submissions', 'instructorled_training', 'government_liaison', 'dcf', 'quantitative_analytics', 'teamoriented', 'series_7', 'leveraged_finance', 'po', 'supplier_negotiation', 'private_banking', 'fx_options', 'umts', 'nodejs', 'private_investigations', 'wire_framing', 'winforms', '360_feedback', 'sqlite', 'maritime_operations', 'construction_documents', 'interior_architecture', 'autocad_architecture', 'multichannel_retail', 'community_organizing', 'adult_education', 'focus_groups', 'fiber_optics', 'psychometrics', 'coordinating_events', 'spaceplanning', 'vmware_esx', 'interpersonal_skill', 'fine_dining', 'capital_iq', 'ingredient', 'digital_printing', 'fixed_annuities', 'temporary_placement', 'brand_architecture', 'ichgcp', 'speech_writing', 'buyer_representation', 'electrician', 'divestiture', 'mineral_exploration', 'geotechnical_engineering', 'cpm_scheduling', 'case_managment', 'tqm', 'information_assurance', 'preventive_maintenance', 'sales_effectiveness', 'journal_entries', 'criminal_investigations', 'sap_bw', 'pharmaceutic', 'military_experience']\n",
      "\n",
      "\n",
      "['construction', 'water', 'energy', 'risk', 'hospitality', 'hotel', 'environmental', 'healthcare', 'information_security', 'emergency', 'transportation', 'commercial', 'foreign_exchange', 'claims', 'supply', 'waste', 'facilities', 'wealth', 'demand', 'carbon']\n",
      "\n",
      "\n",
      "['accounting', 'technical', 'support', 'knowledge', 'reporting']\n",
      "\n",
      "\n",
      "['sql_server', 'visual_studio', 'sql', 'mac_os', 'x', 'studio', 'windows_server', 'microsoft_sql', 'pro', '2008']\n",
      "\n",
      "\n",
      "['microsoft', 'excel', 'word', 'powerpoint', 'access', 'ms']\n",
      "\n",
      "\n",
      "['procurement', 'pipeline', 'hospital', 'banking', 'teams', 'retail', 'consultancy', 'industrial', 'building', 'travel', 'plans', 'new_home', 'plan']\n",
      "\n",
      "\n",
      "['rd', 'contract_negotiation', 'loan', 'commodity', 'customer_satisfaction', 'sale', 'mortgage', 'bond', 'funding', 'corporate_finance', 'airline', 'joint_ventures', 'cost_accounting', 'ifrs', 'strategic_thinking', 'hedge_funds', 'upstream', 'risk_assessment', 'adult', 'derivative', 'investor_relations', 'client_services', 'information_technology', 'economic_development', 'assortment', 'reinsurance', 'public_policy', 'corporate_law', 'broker', 'employee_benefits', 'retirement_planning', 'strategies', 'renovation', 'criminal_law', 'lease', 'underwriting', 'professional_liability', 'workers_compensation', 'liability', 'disaster_recovery', 'costing', 'international_trade', 'export', 'purchasing', 'trend', 'mi', 'building_materials', 'transfer_pricing', 'market_entry', 'automobile', 'structuring', 'sport', 'professional_services', 'arbitration', 'income_tax', 'trust', 'superannuation', 'life_insurance', 'direct_mail', 'general_ledger', 'condo', 'actuarial', 'option', 'fx', 'stewardship', 'family_law', 'stock', 'mixeduse', 'hedging', 'billing', 'treatment', 'bankruptcy', 'human_rights', 'mergers_acquisitions', 'seller', 'dod', 'investment_advisory', 'reconciliation', 'claim', 'gaap', 'agriculture', 'sampling', 'protection', 'purchase', 'economic', 'positioning', 'charity', 'benchmarking', 'retirement', 'shipping', 'local_government', 'mineral', 'high_net', 'reo', 'longterm_care', 'temporary_staffing', 'rental', 'commercial_lines', 'consolidation', 'act', 'personal_injury', 'appraisal']\n",
      "\n",
      "\n",
      "['integrated_marketing', 'business', 'merger', 'ip', 'social_networking', 'public_relations', 'market', 'financial', 'personal', 'marketing_communications', 'sales_operations', 'intellectual_property', 'business_continuity', 'capacity', 'buying', 'resource', 'international_tax', 'career', 'organizing', 'land_use', 'human_resource', 'cost_reduction', 'corporate_tax', 'job_search', 'territory', 'pension', 'inheritance_tax', 'wedding', 'patent']\n",
      "\n",
      "\n",
      "['analysis', 'applicant_tracking', 'networking', 'analytics', 'integration', 'payment', 'wireless', 'integrated', 'based', 'email', 'biology', 'identity', 'electronic', 'messaging', 'camera', 'imaging']\n",
      "\n",
      "\n",
      "['photoshop', 'dreamweaver', 'illustrator', 'suite', 'indesign', 'flash', 'adobe_photoshop', 'adobe']\n",
      "\n",
      "\n",
      "['talent', 'team', 'personnel', 'executive', 'people', 'crisis', 'partnership', 'stakeholder', 'volunteer', 'sports', 'partner', 'staff', 'conflict']\n",
      "\n",
      "\n",
      "['project', 'hris', 'supply_chain', 'operations', 'production', 'inventory', 'configuration', 'release', 'manufacturing', 'asset', 'warehouse', 'logistics', 'store', 'distribution', 'document', 'projects', 'workflow']\n",
      "\n",
      "\n",
      "['process', 'implementation', 'control']\n",
      "\n",
      "\n",
      "['change', 'organizational', 'presentation', 'information', 'communication', 'learning', 'thinking']\n",
      "\n",
      "\n",
      "['javascript', 'html', 'php', 'api', 'net', 'aspnet', 'c', 'java', 'python', 'web_services', 'vbnet', 'jquery', 'ajax', 'xml', 'visual_basic', 'scripting', 'mysql', 'embedded', 'spring', 'jsp', 'j2ee', 'programming', 'perl', 'asp', 'html_css', 'developer', 'mvc', 'basic', 'cc']\n",
      "\n",
      "\n",
      "['tendering', 'employment_law', 'telecommunication', 'oil', 'biotechnology', 'sop', 'gas', 'natural_gas', 'resort', 'internal_audit', 'presales', 'haccp', 'catering', 'food_beverage', 'soa', 'aerospace', 'credit_risk', 'sarbanesoxley', 'internal_controls', 'sox', 'assurance', 'executive_search', 'tpm', 'petroleum', 'onshore', 'inventory_control', 'defence', 'risk_mitigation', 'chemistry', 'defense', 'front_office', 'leed', 'prepress', 'aviation', 'smart_grid', 'semiconductor', 'wholesale', 'chemical', 'regulatory_affairs', 'food_safety', 'warehousing', 'dispute_resolution', 'climate_change', 'supervision', 'machining', 'quality_system', 'food_service', 'informatics', 'rail', 'coal', 'civil', 'court', 'surveillance', 'medicine', 'wellness', 'transmission', 'rooms_division', 'iso', 'occupational_health', 'contractor', 'logging', 'international_law', 'nuclear', 'marine_engineering', 'water_resources', 'port', 'public_health', 'petrochemical', 'gmp', 'cgmp', 'fda', 'ehs', 'health_safety', 'fitness', 'footwear', 'maritime', 'nursing', 'nutrition', 'regulatory_requirements', 'factory', 'menu_development', 'leisure', 'road', 'cosmetic', 'banquet', 'green', 'soil', 'remediation']\n",
      "\n",
      "\n",
      "['monitoring', 'management', 'managment', 'audits', 'auditing', 'compliance', 'governance', 'audit', 'assessment', 'review', 'standards', 'measurement', 'evaluation', 'documentation', 'controls', 'center', 'level', 'preparation']\n",
      "\n",
      "\n",
      "['compensation', 'tax', 'commercial_real', 'estate']\n",
      "\n",
      "\n",
      "['contract', 'medical', 'human', 'government', 'internal', 'board', 'regulatory', 'public', 'pharmaceutical', 'fraud', 'legal']\n",
      "\n",
      "\n",
      "['clinical', 'turn_around', 'restaurant', 'vendor', 'interim', 'pl', 'cash', 'cash_flow', 'reservoir', 'stress', 'treasury', 'transition', 'time', 'operational_risk', 'enterprise_risk', 'product_lifecycle', 'master_data', 'revenue', 'service_delivery', 'supplier', 'print_production', 'interest_rate', 'enterprise_content', 'revenue_cycle', 'supplier_quality', 'pain', 'issues', 'yield', 'incident', 'working_capital', 'earned_value', 'records', 'expense', 'order', 'reputation', 'stage', 'stormwater']\n",
      "\n",
      "\n",
      "['strategic']\n",
      "\n",
      "\n",
      "['adobe_creative', 'microsoft_excel', 'microsoft_office', 'ms_project', 'outlook', 'microsoft_word', 'ms_office', 'visio', 'oracle_ebusiness', '2003', '2007', '2010']\n",
      "\n",
      "\n",
      "['development', 'strategy', 'design', 'international', 'process_improvement', 'innovation', 'requirements', 'process_mapping', 'global', 'cards', 'process_reengineering', 'sustainable', 'culture', 'model', 'china']\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cluster import AgglomerativeClustering\n",
    "\n",
    "# Print \"skills\" clusters\n",
    "n_clusters = n_cluster_2\n",
    "cluster = AgglomerativeClustering(n_clusters=n_clusters, affinity='euclidean', linkage='ward') \n",
    "\n",
    "model2vec_list = list()\n",
    "for i in list(model2.wv.vocab.keys()):\n",
    "    model2vec_list.append(model2[i])\n",
    "model2vec_cluster = list(cluster.fit_predict(model2vec_list))\n",
    "\n",
    "model2vec = dict()\n",
    "for i in range(len(model2.wv.vocab.keys())):\n",
    "    model2vec[list(model2.wv.vocab.keys())[i]] = model2vec_cluster[i]\n",
    "print('\\n')\n",
    "model2_cluster_list = list()\n",
    "for i in range(0,n_clusters):\n",
    "    print([k for k,v in model2vec.items() if v == i])\n",
    "    model2_cluster_list.append([k for k,v in model2vec.items() if v == i])\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "['history', 'english', 'general_studies', 'art', 'biology', 'literature', 'philosophy', 'german', 'spanish', 'english_literature', 'french', 'music', 'liberal_arts', 'geography', 'english_language']\n",
      "\n",
      "\n",
      "['public_relations', 'communication', 'communications', 'arts', 'journalism', 'advertising']\n",
      "\n",
      "\n",
      "['computer_science', 'science', 'applied_mathematics', 'sciences', 'computing', 'informatics']\n",
      "\n",
      "\n",
      "['electronic_engineering', 'systems', 'electronics_communication', 'computer', 'information', 'engineering', 'electrical_engineering', 'technology', 'telecommunication', 'software_engineering', 'telecommunications', 'electronics', 'electrical', 'software']\n",
      "\n",
      "\n",
      "['education', 'public_administration', 'development', 'administration', 'criminal_justice', 'social_work', 'relations', 'health', 'policy', 'nursing']\n",
      "\n",
      "\n",
      "['medicine', 'geology', 'chemistry', 'applied', 'biochemistry', 'environmental', 'microbiology']\n",
      "\n",
      "\n",
      "['marketing', 'management', 'strategy', 'finance', 'strategic_management', 'mba', 'information_systems', 'operations', 'financial', 'entrepreneurship', 'supply_chain']\n",
      "\n",
      "\n",
      "['human', 'social']\n",
      "\n",
      "\n",
      "['human_resources', 'business', 'business_administration', 'international_business', 'international', 'strategic', 'leadership', 'global', 'human_resource', 'hr', 'real_estate']\n",
      "\n",
      "\n",
      "['psychology', 'anthropology', 'major', 'minor', 'art_history', 'sociology']\n",
      "\n",
      "\n",
      "['chemical_engineering', 'production', 'biomedical_engineering', 'petroleum_engineering', 'mechanical_engineering', 'industrial', 'civil_engineering', 'biotechnology', 'industrial_design', 'industrial_engineering', 'mechanical', 'pharmacy']\n",
      "\n",
      "\n",
      "['mathematics', 'physics', 'mathematics_physics', 'maths', 'math']\n",
      "\n",
      "\n",
      "['emphasis', 'school', 'information_technology', 'concentration', 'physic', 'construction_management', 'planning', 'statistic', 'analysis', 'master', 'research', 'construction']\n",
      "\n",
      "\n",
      "['general', 'services']\n",
      "\n",
      "\n",
      "['electrical_electronics']\n",
      "\n",
      "\n",
      "['media', 'animation', 'graphic_design', 'design', 'architecture', 'photography', 'interior_design', 'multimedia']\n",
      "\n",
      "\n",
      "['economics', 'statistics', 'accounting', 'accountancy', 'law', 'taxation', 'commerce', 'accounting_finance', 'banking']\n",
      "\n",
      "\n",
      "['advanced', 'program', 'course', 'training', 'diploma', 'certificate', 'high_school', 'project_management', 'professional']\n",
      "\n",
      "\n",
      "['physics_chemistry']\n",
      "\n",
      "\n",
      "['political_science', 'studies', 'government', 'international_relations', 'politics']\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print \"major\" clusters\n",
    "n_clusters = n_cluster_3\n",
    "cluster = AgglomerativeClustering(n_clusters=n_clusters, affinity='euclidean', linkage='ward') \n",
    "\n",
    "model3vec_list = list()\n",
    "for i in list(model3.wv.vocab.keys()):\n",
    "    model3vec_list.append(model3[i])\n",
    "model3vec_cluster = list(cluster.fit_predict(model3vec_list))\n",
    "\n",
    "model3vec = dict()\n",
    "for i in range(len(model3.wv.vocab.keys())):\n",
    "    model3vec[list(model3.wv.vocab.keys())[i]] = model3vec_cluster[i]\n",
    "print('\\n')\n",
    "model3_cluster_list = list()\n",
    "for i in range(0,n_clusters):\n",
    "    print([k for k,v in model3vec.items() if v == i])\n",
    "    model3_cluster_list.append([k for k,v in model3vec.items() if v == i])\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "['delhi university', 'university of madras', 'symbiosis institute of management studies', 'kendriya vidyalaya', 'indian institute of management bangalore', 'university of pune', 'indian institute of management calcutta', 'indira gandhi national open university', 'indian institute of management ahmedabad']\n",
      "\n",
      "\n",
      "['university of nottingham', 'university of exeter', 'liverpool john moores university', 'university of newcastleupontyne', 'university of warwick  warwick business school', 'cardiff university  prifysgol caerdydd', 'university of leicester', 'the university of manchester', 'the manchester metropolitan university', 'the university of reading', 'loughborough university', 'university of southampton', 'aston university', 'the university of birmingham', 'university of liverpool', 'university of the west of england', 'chartered institute of marketing', 'the open university_the open university']\n",
      "\n",
      "\n",
      "['king saud university', 'american university of beirut', 'universiteit gent', 'imperial college london_imperial college london', 'university of the witwatersrand', 'obafemi awolowo university', 'hult international business school', 'universiteit antwerpen', 'alexandria university', 'ain shams university', 'university of cambridge_university of cambridge', 'erasmus universiteit rotterdam', 'oxford university', 'katholieke universiteit leuven_katholieke universiteit leuven', 'budapesti corvinus egyetem', 'rsm erasmus university', 'escp europe', 'rijksuniversiteit groningen', 'université libre de bruxelles', 'université panthéon sorbonne paris i']\n",
      "\n",
      "\n",
      "['purdue university', 'north carolina state university', 'michigan state university', 'georgia institute of technology', 'the ohio state university', 'indiana university bloomington', 'university of wisconsinmadison', 'miami university']\n",
      "\n",
      "\n",
      "['university of the philippines', 'george mason university  school of management', 'university of minnesotaduluth', 'millersville university of pennsylvania', 'northern michigan university', 'saint leo university', 'savannah college of art and design', 'american intercontinental university', 'grand canyon university', 'thomas edison state college', 'hamline university', 'oklahoma city university', 'franklin university', 'southern illinois university edwardsville', 'kean university', 'texas state universitysan marcos', 'stony brook university', 'johnson and wales university', 'fashion institute of technology', 'assumption college', 'wentworth institute of technology', 'art institute of pittsburgh', 'elon university', 'university of colorado at boulder  leeds school of business', 'yeshiva university', 'richard stockton college of new jersey', 'virginia polytechnic institute and state university  pamplin college of business', 'bates college', 'state university of new york college at oneonta', 'western governors university', 'richmond the american international university in london', 'university of miami', 'university of baltimore', 'tel aviv university', 'st cloud state university', 'cardinal stritch university', 'university of wisconsinoshkosh', 'the johns hopkins university  paul h nitze school of advanced international studies sais', 'austin community college', 'kaplan university', 'loyola college in maryland', 'saint marys university of minnesota', 'babson college', 'robert morris university', 'bellevue university', 'high school', 'university of minnesota', 'texas am university  mays business school', 'institut teknologi bandung itb', 'the university of texas at dallas  school of management', 'harvard university kennedy school of government', 'quinnipiac university', 'california state universityhayward', 'mount holyoke college', 'university of puget sound', 'mercer university', 'the university of dallas', 'university of houston ct bauer college of business', 'wuhan university', 'santa monica college', 'western kentucky university', 'st edwards university', 'university of south carolina  the moore school of business', 'university of oregon  charles h lundquist college of business', 'iona college', 'universidad de salamanca', 'ashford university', 'loyola university new orleans', 'texas christian university  mj neeley school of business', 'stetson university', 'towson university', 'state university of new york college at oswego', 'eastern kentucky university', 'university of north carolina at greensboro', 'rowan university', 'state university of new york at buffalo  school of management', 'northeastern illinois university', 'the university of texas at austin  the red mccombs school of business', 'philadelphia university', 'kennesaw state university', 'florida state university  college of business', 'columbia southern university', 'saint josephs university  erivan k haub school of business', 'school of visual arts', 'canisius college', 'southern connecticut state university', 'university of new hampshire  whittemore school of business and economics', 'college of charleston', 'ohio wesleyan university', 'palomar college', 'lewis and clark college', 'university of the pacific', 'university of california los angeles_university of california los angeles', 'sonoma state university', 'mit', 'barry university', 'university of saint thomas  school of business', 'northern virginia community college', 'university of louisville', 'state university of new york college at fredonia', 'northwood university', 'gemological institute of america', 'hampton university', 'valdosta state university', 'north dakota state university', 'fundação getúlio vargas', 'university of wisconsinstevens point', 'itt technical institute', 'kennesaw state university  michael j coles college of business', 'state university of new york at new paltz', 'western new england college', 'jilin university', 'houston community college', 'gonzaga university', 'fullerton college', 'university of central florida  college of business administration', 'missouri state university', 'bangladesh university of engineering and technology', 'university of santo tomas', 'polytechnic university', 'university of pittsburgh  joseph m katz graduate school of business', 'portland state university  school of business', 'slippery rock university of pennsylvania', 'long island university cw post campus', 'university of tulsa', 'university of south alabama', 'st petersburg college', 'gannon university', 'norwich university', 'southern polytechnic state university', 'weber state university', 'university of evansville', 'george fox university', 'usc', 'central connecticut state university', 'university of scranton', 'muhlenberg college', 'bowdoin college', 'roger williams university', 'eastern washington university', 'huazhong university of science and technology', 'university of missouricolumbia college of business', 'salisbury university', 'de anza college', 'orange coast college', 'stephen f austin state university', 'liberty university', 'the university of connecticut school of business', 'university of wisconsinla crosse', 'trinity university', 'william paterson university of new jersey', 'dale carnegie', 'texas womans university', 'saint josephs university', 'the university of texas at el paso', 'ferris state university', 'sun yatsen university', 'beijing foreign studies university', 'nankai university', 'radford university', 'murray state university', 'mit sloan school of management', 'vanderbilt university  owen graduate school of management', 'university of nevadareno', 'university of alabama in huntsville', 'texas tech university  rawls college of business', 'tianjin university', 'gustavus adolphus college']\n",
      "\n",
      "\n",
      "['city university of new yorkbaruch college', 'bellevue community college', 'university of massachusetts at lowell', 'university of phoenix_university of phoenix', 'pennsylvania state university', 'suffolk university', 'middle tennessee state university', 'park university', 'the university of tennessee at chattanooga', 'carnegie mellon university_carnegie mellon university', 'university of illinois at urbanachampaign  college of business', 'university of nebraska at omaha', 'rice university  jesse h jones graduate school of management', 'colorado technical university', 'rutgers university', 'shippensburg university of pennsylvania', 'university of la verne', 'miami dade college', 'boston university  school of management', 'virginia tech', 'university of missourirolla', 'baylor university', 'grand valley state university', 'university of florida  warrington college of business', 'eastern illinois university', 'youngstown state university', 'fairleigh dickinson university', 'city university of new york city college', 'the university of texas at san antonio', 'southern new hampshire university', 'university of maryland', 'georgia college and state university', 'ithaca college', 'florida international university', 'university of maryland  robert h smith school of business', 'adelphi university', 'northern arizona university', 'fashion institute of design  merchandising', 'excelsior college', 'seton hall university', 'state university of new york at albany', 'mississippi state university', 'san francisco state university college of business', 'university of georgia  terry college of business', 'baldwinwallace college', 'clark university', 'california state universityeast bay', 'university of massachusetts dartmouth', 'university of colorado', 'city university of new yorkhunter college', 'universidad complutense de madrid', 'baruch college', 'the university of memphis', 'berklee college of music', 'new york institute of technologyold westbury', 'wichita state university', 'western washington university', 'montana state universitybozeman', 'university of illinois', 'humboldt state university', 'keller graduate school of management', 'university of west florida', 'city university of new yorkbaruch college  zicklin school of business', 'northern illinois university  college of business', 'georgia institute of technology_georgia institute of technology', 'central washington university', 'university of notre dame  mendoza college of business', 'cornell university_cornell university', 'university of northern iowa', 'east carolina university', 'university of florida_university of florida', 'boise state university', 'pasadena city college', 'saint louis university', 'indiana university of pennsylvania', 'university of northern colorado', 'the university of montana', 'columbia university', 'florida international university  college of business administration', 'texas christian university', 'university of south florida  college of business administration', 'university of new orleans', 'gettysburg college', 'university of michigan_university of michigan', 'university of massachusetts boston', 'the american college', 'allegheny college', 'georgia institute of technology  georgia tech college of management', 'teachers college of columbia university', 'university of wisconsinmadison_university of wisconsinmadison', 'california state universityfullerton  college of business and economics', 'university of southern mississippi', 'john carroll university', 'lawrence technological university', 'community college of the air force', 'university of mississippi', 'indiana university', 'louisiana tech university', 'bentley college  elkin b mccallum graduate school of business', 'state university of new york college at brockport', 'furman university', 'lesley university', 'national chiao tung university', 'college of dupage', 'city university of new york', 'western illinois university', 'university of denver  daniels college of business', 'providence college', 'west chester university of pennsylvania', 'university of richmond', 'manhattan college', 'florida agricultural and mechanical university', 'california university of pennsylvania', 'benedictine university', 'york college of pennsylvania', 'michigan state university_michigan state university', 'lindenwood university', 'davidson college', 'colorado school of mines', 'university of missourikansas city', 'nationallouis university', 'tennessee technological university', 'purdue university_purdue university', 'fordham university  graduate school of business administration', 'valparaiso university', 'the citadel']\n",
      "\n",
      "\n",
      "['rhodes university', 'university of luton', 'university of auckland_university of auckland', 'king fahd university of petroleum  minerals', 'national university of science and technology', 'ned university of engineering and technology', 'universiteit gent_universiteit gent', 'university of pretoriauniversiteit van pretoria', 'university of abertay dundee', '청강아카데미학원', 'massey university', 'the university of edinburgh_the university of edinburgh', 'university of cape town_university of cape town', 'academia de studii economice din bucurești', 'damelin', 'university of kwazulunatal', 'vrije universiteit brussel', 'university of bahrain', 'karachi university', 'institute of business administration', 'university of the punjab lahore', 'edith cowan university', 'edhec business school', 'university of paisley', 'sveuèilište u zagrebu', 'politecnico di milano', 'universiteit van amsterdam', 'singapore management university', 'instituto superior técnico', 'nyenrode business universiteit', 'city university of hong kong', 'aberdeen college', 'univerza v ljubljani', 'kwame nkrumah university of science and technology kumasi', 'aquinas college', 'the hong kong polytechnic university', 'tel aviv university_tel aviv university', 'auckland university of technology', 'the university of dundee', 'kyiv national taras shevchenko university', 'new mexico state university', 'universiteit utrecht', 'eötvös loránd tudományegyetem', 'hanzehogeschool groningen', 'liceo scientifico', 'microsoft', 'technische universiteit delft', 'ngee ann polytechnic', 'università di bologna', 'solvay business school', 'waterford institute of technology', 'university of nairobi', 'vlerick leuvengent management school', 'universidade paulista', 'university of strathclyde_university of strathclyde', 'university of johannesburg', 'the university of northampton', 'university of stellenbosch', 'universitat ben gurion banegev', 'università degli studi di padova', 'university of karachi', 'trinity college dublin_trinity college dublin', 'stellenbosch universityuniversiteit stellenbosch', 'haagse hogeschoolth rijswijk', 'university of zimbabwe']\n",
      "\n",
      "\n",
      "['macquarie university', 'curtin university of technology', 'university of melbourne', 'rmit university', 'southern cross university', 'the university of western australia', 'university of south australia', 'the university of queensland', 'australian graduate school of management', 'finsia', 'deakin university', 'university of sydney', 'university of new south wales', 'university of newcastle', 'queensland university of technology', 'securities institute of australia', 'cpa australia', 'university of adelaide', 'australian institute of management', 'swinburne university of technology', 'the australian national university', 'griffith university', 'macquarie university  graduate school of business', 'university of new england au', 'charles sturt university', 'australian institute of company directors', 'unsw', 'tafe_tafe']\n",
      "\n",
      "\n",
      "['rajiv gandhi prodyogiki vishwavidyalaya', 'sardar patel university', 'columbia university  fu foundation school of engineering and applied science', 'bharati vidyapeeth', 'apeejay school', 'lahore university of management sciences', 'national institute of fashion technology', 'mumbai university', 'doctor babasaheb ambedkar marathwada university', 'govind ballabh pant krishi evam praudyogik vishwavidyalaya', 'national institute of technology calicut', 'institute of technology bhu', 'national institute of technology warangal', 'vellore institute of technology', 'symbiosis institute of business management', 'maharashtra institute of technology', 'motilal nehru national institute of technology', 'louisiana state university and agricultural and mechanical college', 'karnatak university', 'pune institute of computer technology', 'b m s college of engineering', 'cochin university of science and technology', 'visvesvaraya national institute of technology', 'modern school', 'jamnalal bajaj institute of management studies', 'mahatma gandhi university', 'sri venkateswara university', 'delhi public school', 'lamar university', 'punjab engineering college', 'university of mysore', 'international institute of information technology', 'utkal university', 'school of planning and architecture', 'university of pune_university of pune', 'mangalore university', 'tata institute of social sciences', 'madras university', 'manipal institute of technology', 'maulana azad national institute of technology', 'mithibai college', 'bharathiar university', 'west bengal university of technology', 'university of mumbai_university of mumbai', 'indian school of mines', 'the maharaja sayajirao university of baroda', 'university of michigan  stephen m ross school of business_university of michigan', 'guru nanak dev university', 'st francis college', 'amravati university', 'calcutta university', 'psg college of technology', 'ibs hyderabad']\n",
      "\n",
      "\n",
      "['georgia state university', 'keller graduate school of management of devry university', 'university of cincinnati', 'the university of connecticut', 'central michigan university', 'stanford university graduate school of business', 'boston university', 'university of missouricolumbia', 'university of minnesotatwin cities', 'west virginia university', 'state university of new york at buffalo', 'depaul university', 'western michigan university', 'university of maryland university college', 'the university of georgia', 'marquette university', 'wake forest university', 'illinois institute of technology', 'southern illinois university carbondale', 'iowa state university', 'rochester institute of technology', 'california state universitynorthridge', 'webster university', 'university of tennesseeknoxville', 'university of south carolinacolumbia', 'university of illinois at chicago', 'university of south florida', 'northern illinois university', 'florida state university', 'colorado state university', 'university of dayton', 'florida institute of technology', 'widener university', 'university of oklahoma', 'illinois state university', 'university of hawaii at manoa', 'tulane university', 'university of utah', 'university of central florida', 'virginia polytechnic institute and state university', 'ohio university', 'oregon state university', 'wayne state university', 'depaul university  charles h kellstadt graduate school of business', 'cleveland state university', 'golden gate university', 'kent state university', 'dartmouth college  the tuck school of business at dartmouth', 'regis university', 'xavier university', 'university of colorado at denver', 'university of north carolina at charlotte', 'columbia college chicago', 'university of nebraskalincoln', 'university of arkansas at fayetteville', 'indiana wesleyan university', 'howard university', 'auburn university', 'united states air force academy', 'virginia commonwealth university', 'embryriddle aeronautical university', 'georgia state university  j mack robinson college of business', 'thunderbird school of global management', 'bowling green state university', 'portland state university', 'southern methodist university', 'the george washington university law school']\n",
      "\n",
      "\n",
      "['san jose state university', 'university of southern california', 'stanford university', 'brigham young university', 'university of california davis', 'tufts university', 'stanford university_stanford university', 'university of pennsylvania  the wharton school', 'university of california santa barbara', 'university of california los angeles', 'carnegie mellon university', 'university of washington', 'san diego state universitycalifornia state university', 'santa clara university', 'university of california santa cruz', 'university of california berkeley  walter a haas school of business', 'pepperdine university', 'peking university', 'fudan university']\n",
      "\n",
      "\n",
      "['imd international institute for management development  business programs', 'pmi', 'la trobe university', 'university of western sydney', 'rmit university_rmit university', 'rmit', 'queensland university of technology_queensland university of technology', 'university of ballarat', 'university of technology sydney_university of technology sydney', 'st josephs college', 'bond university', 'monash university_monash university', 'victoria university of wellington', 'university of sydney_university of sydney', 'university of melbourne  melbourne business school', 'college of engineering trivandrum', 'the university of western australia_the university of western australia', 'university of canberra', 'james cook university', 'isaca', 'university of tasmania', 'delhi vishwavidyalaya', 'university of new england', 'university of waikato', 'australian catholic university', 'the university of queensland_the university of queensland', 'sydney university']\n",
      "\n",
      "\n",
      "['university of hertfordshire', 'university of portsmouth', 'leeds metropolitan university', 'university of strathclyde', 'university of central lancashire', 'sheffield hallam university', 'oxford brookes university', 'queen mary u of london', 'university of surrey', 'coventry university', 'middlesex university', 'keele university', 'institute of chartered accountants in england and wales', 'university of westminster', 'university of bradford', 'brunel university', 'kingston university', 'cranfield university', 'the chartered institute of management accountants', 'city university gb', 'cass business school', 'de montfort university', 'the university of sheffield', 'the nottingham trent university', 'lancaster university', 'the university of hull', 'university of wales cardiff', 'cranfield university  cranfield school of management', 'university of sunderland', 'london south bank university', 'college of law']\n",
      "\n",
      "\n",
      "['the university of texas at arlington', 'jadavpur university', 'indian institute of technology bombay', 'indian institute of technology madras', 'indian institute of technology delhi', 'indian institute of technology guwahati', 'jawaharlal nehru technological university', 'university of bridgeport', 'california state universitysacramento', 'r v college of engineering bangalore', 'the university of texas at austin_the university of texas at austin', 'malaviya national institute of technology', 'indian institute of technology kharagpur', 'university of maryland baltimore county', 'nagarjuna university', 'university of maryland college park_university of maryland college park', 'college of engineering pune', 'texas am university_texas am university']\n",
      "\n",
      "\n",
      "['university of windsor', 'national university of singapore', 'concordia university', 'project management institute', 'harvard business school', 'copenhagen business school', 'memorial university of newfoundland', 'university of toronto at scarborough', 'university of ottawa  université dottawa', 'university of manitoba', 'the university of british columbia_the university of british columbia', 'brock university', 'british columbia institute of technology', 'seneca college of applied arts and technology', 'humber college', 'saint marys university', 'university of saskatchewan', 'the chinese university of hong kong', 'centennial college', 'georgian college', 'laurentian universityuniversité laurentienne', 'athabasca university', 'université laval', 'queens university_queens university', 'royal roads university', 'durham college', 'st francis xavier university', 'lakehead university', 'conestoga college']\n",
      "\n",
      "\n",
      "['eastern michigan university', 'st marys university', 'truman state university', 'california polytechnic state universitysan luis obispo', 'california state universityfresno', 'ball state university', 'university of north texas', 'california state universityfullerton', 'university of minnesotatwin cities_university of minnesotatwin cities', 'california state universitylos angeles', 'florida atlantic university', 'bryant university', 'university of alabama', 'washington state university', 'state university of new york college at buffalo', 'university of nevadalas vegas', 'northern kentucky university', 'university of michigandearborn', 'indiana universitypurdue university at indianapolis', 'oakland university', 'rhode island school of design', 'university of california irvine', 'metropolitan state college of denver', 'st marys college of california', 'james madison university', 'hofstra university', 'walsh college of accountancy and business administration', 'university of delaware  lerner college of business and economics', 'university of wisconsinmilwaukee', 'capella university', 'university of colorado at colorado springs', 'walden university', 'bradley university', 'devry university', 'art center college of design', 'university of san diego', 'university of missourisaint louis', 'academy of art university', 'washington university in st louis  olin business school', 'san francisco state university', 'chapman university', 'full sail university', 'rutgers the state university of new jerseynewark', 'montclair state university', 'university of detroit mercy', 'university of washington_university of washington', 'university of wyoming', 'california state universitychico', 'university of hartford', 'university of california riverside', 'university of new hampshire', 'university of houstonclear lake', 'california college of the arts', 'university of idaho', 'drake university', 'university of minnesota  carlson school of management', 'duquesne university', 'national university', 'purdue university  krannert school of management', 'university of wisconsineau claire', 'wake forest university  babcock graduate school of management', 'st olaf college', 'stevens institute of technology', 'emerson college', 'nova southeastern university', 'foothill college', 'utah state university', 'loyola marymount university', 'southern methodist university  cox school of business', 'la salle university', 'university of redlands', 'the johns hopkins university  carey business school', 'roosevelt university', 'troy university', 'michigan technological university', 'appalachian state university', 'augustana college', 'university of wisconsinmadison  school of business', 'creighton university', 'occidental college', 'zhejiang university']\n",
      "\n",
      "\n",
      "['university of east london', 'university of durham', 'university of kent', 'university of aberdeen', 'royal college of art', 'university of manchester  institute of science and technology', 'london metropolitan university', 'university of sussex', 'university of plymouth', 'university of brighton', 'university of wales bangor', 'university of essex', 'staffordshire university', 'southampton solent university', 'the university of salford', 'university of manchester  manchester business school', 'northumbria university', 'thames valley university', 'university of greenwich', 'university of bristol', 'trinity college dublin', 'heriotwatt university', 'university of wales aberystwyth', 'the university of huddersfield', 'the robert gordon university', 'anglia ruskin university', 'bpp law school', 'the university of wolverhampton', 'the university of glasgow', 'chartered institute of management accountants', 'birmingham city university', 'henley management college', 'open university', 'birkbeck u of london', 'royal holloway u of london']\n",
      "\n",
      "\n",
      "['the university of calgary_the university of calgary', 'university of toronto at mississauga  erindale college', 'university of waterloo_university of waterloo', 'mount allison university', 'mcmaster university_mcmaster university', 'northern alberta institute of technology', 'bcit', 'universitatea „alexandru ioan cuza” din iași', 'université de sherbrooke', 'southern alberta institute of technology', 'kwantlen university college', 'fanshawe college', 'capilano university', 'mohawk college', 'grant macewan college', 'ryerson university_ryerson university', 'université de montréal', 'algonquin college of applied arts and technology', 'mount royal college', 'helsingin yliopisto', 'trent university', 'the university of lethbridge', 'red river college', 'the university of winnipeg', 'southern alberta institute of technology  sait']\n",
      "\n",
      "\n",
      "['university of pennsylvania', 'yale university', 'cornell university', 'massachusetts institute of technology', 'university of california berkeley', 'new york university', 'massachusetts institute of technology_massachusetts institute of technology']\n",
      "\n",
      "\n",
      "['university of virginia', 'the university of texas at austin', 'university of arizona', 'university of michigan', 'university of colorado at boulder', 'washington university in st louis', 'northwestern university', 'duke university', 'university of michigan  stephen m ross school of business', 'syracuse university', 'the george washington university']\n",
      "\n",
      "\n",
      "['london school of economics and political science', 'university of oxford', 'university of cambridge']\n",
      "\n",
      "\n",
      "['university of phoenix', 'university of north carolina at chapel hill  kenanflagler business school', 'the johns hopkins university', 'university of illinois at urbanachampaign', 'university of maryland college park', 'loyola university of chicago', 'northeastern university', 'university of notre dame', 'penn state university', 'dartmouth college', 'texas am university', 'rutgers the state university of new jerseynew brunswick', 'university of delaware', 'rensselaer polytechnic institute', 'university of pittsburgh', 'arizona state university', 'university of iowa', 'villanova university', 'university of chicago', 'the george washington university  school of business', 'university of florida', 'university of oregon', 'united states naval academy', 'emory university  goizueta business school', 'university of north carolina at chapel hill', 'drexel university', 'university of kansas', 'university of science and technology of china', 'university of denver', 'temple university', 'the college of william and mary', 'united states military academy at west point']\n",
      "\n",
      "\n",
      "['osmania university', 'visvesvaraya technological university', 'andhra university', 'sp jain institute of management  research', 'indian institute of management lucknow', 'guru gobind singh indraprastha university', 'uttar pradesh technical university', 'faculty of management studies  university of delhi', 'university of rajasthan', 'nagpur university', 'delhi college of engineering', 'shivaji university', 'indian institute of science', 'devi ahilya vishwavidyalaya']\n",
      "\n",
      "\n",
      "['birla institute of technology and science', 'indian institute of technology kanpur', 'university of mumbai', 'indian institute of technology roorkee']\n",
      "\n",
      "\n",
      "['university of derby', 'university of wales swansea', 'national technical university of athens', 'bournemouth university', 'university college dublin', 'city university london', 'association of chartered certified accountants', 'dublin city university', 'university of lincoln', 'napier university', 'university college cork', 'the university of bolton', 'dublin institute of technology', 'the university of stirling', 'university of east anglia', 'canterbury christ church university', 'queens university belfast', 'university of the arts london', 'national university of ireland maynooth', 'university of limerick', 'london school of economics and political science u of london', 'goldsmiths college u of london', 'cork institute of technology', 'university of teesside', 'dublin business school', 'athens university of economics and business', 'the university of glamorgan']\n",
      "\n",
      "\n",
      "['katholieke universiteit leuven']\n",
      "\n",
      "\n",
      "['arizona state university_arizona state university', 'helwan university cairo', 'arab academy for science technology and maritime transport', 'national technical university of ukraine kyiv polytechnic institute', 'technical university sofia', 'university of leeds_university of leeds', 'università degli studi di milano']\n",
      "\n",
      "\n",
      "['the open university']\n",
      "\n",
      "\n",
      "['university of waterloo', 'wilfrid laurier university', 'york university', 'university of guelph', 'sheridan college', 'york university  schulich school of business', 'mcmaster university', 'university of victoria', 'the university of calgary', 'carleton university', 'dalhousie university', 'university of toronto_university of toronto', 'george brown college', 'university of alberta', 'simon fraser university', 'university of new brunswick', 'the university of western ontario  richard ivey school of business', 'canadian securities institute']\n",
      "\n",
      "\n",
      "['gujarat university', 'indian institute of management kozhikode', 'university of allahabad', 'bharathidasan university', 'national institute of information technology', 'shri ram college of commerce', 'institute of chartered accountants of india', 'narsee monjee institute of management studies', 'university of lucknow', 'punjab technical university', 'university of calcutta', 'institute of company secretaries of india', 'maharshi dayanand university', 'welingkar institute of management', 'management development institute', 'netaji subhas institute of technology', 'amity business school', 'pune university', 'university of delhi', 'national institute of technology kurukshetra', 'dav public school', 'fergusson college', 'university of calicut', 'indian institute of planning and management', 'aligarh muslim university', 'jamia millia islamia', 'panjab university', 'manipal academy of higher education', 'xavier institute of management', 'banaras hindu university', 'christ college  bangalore', 'national institute of industrial engineering', 'osmania university_osmania university', 'national institute of technology rourkela', 'national institute of technology tiruchirappalli', 'chaudhary charan singh university', 'icfai', 'niit', 'ignou', 'icai']\n",
      "\n",
      "\n",
      "['university of international business and economics', 'universiteit maastricht', 'teknillinen korkeakoulutekniska högskolan', 'queens college', 'simmons college', 'the interdisciplinary center', 'universidad de granada', 'universitatea „babeșbolyai” din clujnapoca', 'st peters college', 'aljamiah alurdunia', 'jamiat almalik abdulaziz', 'universidad de los andes', 'university of kerala', 'school name', 'university of new haven', 'ateneo de manila university', 'univerzitet u beogradu', 'pace university  lubin school of business', 'rhode island college', 'technionmachon technologi le israel', 'state university of new york empire state college', 'marist college', 'stockholms universitet', 'universiteit van tilburg', 'smith college', 'central queensland university', 'other', 'boston university_boston university', 'minnesota state university moorhead', 'the hebrew university', 'connecticut college', 'sacred heart university', 'siena college', 'universidade federal do rio de janeiro', 'universidad de sevilla', 'university of engineering and technology lahore', 'technische universiteit eindhoven', 'université paris sorbonne paris iv', 'politecnico di torino', 'hogeschool van amsterdam', 'new horizons', 'university of arizona eller college of management', 'university of regina', 'st norbert college', 'college of the holy cross', 'syracuse university  martin j whitman school of management', 'rollins college', 'xian jiaotong university', 'bloomsburg university of pennsylvania', 'hong kong baptist university', 'university of maine', 'university of helsinki']\n",
      "\n",
      "\n",
      "['anna university', 'bangalore university', 'institute of management technology ghaziabad', 'delhi public school  r k puram', 'annamalai university', 'st xaviers college', 'cfa institute usa', 'indian school of business', 'the institute of chartered accountants of india', 'symbiosis international university', 'kurukshetra university', 'indian institute of foreign trade', 'birla institute of technology', 'pondicherry university', 'madurai kamaraj university', 'sikkim manipal university of health medical and technological sciences', 'xlri jamshedpur']\n",
      "\n",
      "\n",
      "['california state polytechnic universitypomona', 'university of san francisco', 'university of california san diego', 'university of massachusetts amherst', 'duke university  the fuqua school of business', 'george mason university', 'lehigh university', 'university of rochester', 'ucla', 'brandeis university', 'university of southern california  marshall school of business', 'georgetown university law center', 'pratt institute', 'carnegie mellon university  tepper school of business', 'university of california los angeles  the anderson school of management', 'worcester polytechnic institute', 'harvard university graduate school of education', 'emory university', 'rice university', 'university of illinois at urbanachampaign_university of illinois at urbanachampaign', 'kettering university', 'california state universitylong beach', 'cornell university  sc johnson graduate school of management', 'case western reserve university', 'ucla anderson school of management', 'pepperdine university the george l graziadio school of business and management', 'new school university', 'vanderbilt university', 'university of washington michael g foster school of business', 'babson college  franklin w olin graduate school of business', 'california institute of technology', 'parsons school of design', 'the catholic university of america', 'state university of new york at binghamton', 'clarkson university', 'seattle university', 'new york university school of law', 'middlebury college', 'university of pennsylvania law school', 'university of california san francisco']\n",
      "\n",
      "\n",
      "['beijing language and culture university', 'università degli studi di roma la sapienza', 'university of ibadan', 'university of oxford  said business school', 'asian institute of technology', 'iese business school  university of navarra', 'ie instituto de empresa', 'sda bocconi', 'grenoble ecole de management  grenoble graduate school of business', 'institut detudes politiques de paris', 'universitatea „politehnica” din bucurești', 'universiteit twente', 'lunds universitet', 'hec school of management', 'universidade de são paulo', 'amirkabir university of technology  tehran polytechnic', 'university of otago', 'university of new south wales_university of new south wales', 'université pierre et marie curie paris vi', 'università commerciale luigi bocconi', 'université paris sud paris xi', 'lebanese american university', 'vrije universiteit amsterdam', 'esade business school', 'københavns universitet', 'shanghai university', 'jamiat almalik saud', 'american university of sharjah']\n",
      "\n",
      "\n",
      "['northwestern university  kellogg school of management', 'boston university school of law', 'indiana university  kelley school of business', 'bentley university', 'university of vermont', 'new york university  leonard n stern school of business', 'massachusetts institute of technology  sloan school of management', 'union college', 'arizona state university w p carey school of business', 'university of rhode island', 'fairfield university', 'wesleyan university', 'georgetown university  the mcdonough school of business', 'nanjing university', 'claremont mckenna college', 'university of miami  school of business', 'temple university  fox school of business and management', 'macalester college', 'saint lawrence university', 'northeastern university  graduate school of business administration', 'columbia university  school of international and public affairs', 'hong kong university of science and technology  school of business and management', 'colgate university', 'trinity collegehartford', 'carleton college', 'oberlin college', 'pace university', 'pomona college', 'vassar college', 'rider university', 'boston college  wallace e carroll graduate school of management', 'bucknell university', 'amherst college', 'swarthmore college', 'the lawrenceville school']\n",
      "\n",
      "\n",
      "['cairo university', 'school of oriental and african studies u of london', 'university of auckland', 'university of ulster', 'cambridge university', 'university of lagos', 'national university of ireland galway', 'chartered insurance institute', 'kings college london', 'luniversità ta malta', 'university college dublin_university college dublin', 'acca', 'ethnikon kai kapodistriakon panepistimion athinon', 'university of cape town', 'university of st andrews', 'london college of fashion', 'royal holloway university of london', 'university of nottingham  nottingham university business school', 'manchester business school', 'ashridge business school', 'leeds university', 'birkbeck college university of london']\n",
      "\n",
      "\n",
      "['preston university', 'sir syed university of engineering and technology', 'mapua institute of technology', 'hogeschool van utrecht', 'karel de grote hogeschool antwerpen', 'hauniversita hapetuha', 'national university of computer and emerging sciences', 'galwaymayo institute of technology', 'kyiv national economics university', 'buckinghamshire chilterns university college', 'university of jordan', 'vilniaus universitetas', 'kenyatta university', 'budapesti mûszaki és gazdaságtudományi egyetem', 'unisa', 'haute ecole groupe ichec  isc saintlouis  isfsc bruxelles', 'milwaukee area technical college', 'hogeschool gent', 'university of pretoriauniversiteit van pretoria_university of pretoriauniversiteit van pretoria', 'lessius hogeschool antwerpen', 'tshwane university of technology', 'cape peninsula university of technology', 'mba', 'the university of birmingham_the university of birmingham', 'hogeschool rotterdam', 'ravensbourne college of design and communication', 'university of canterbury_university of canterbury', 'hogeschool antwerpen', 'university of belgrade', 'northern alberta institute of technology  nait', 'iqra university']\n",
      "\n",
      "\n",
      "['columbia university  columbia business school', 'santa clara university  leavey school of business', 'haverford college', 'wheaton college', 'suffolk university  sawyer school of management', 'nanyang technological university', 'boston college', 'columbia business school', 'landmark education', 'harvey mudd college', 'the college of new jersey', 'columbia university  graduate school of journalism', 'saint johns university', 'universidad de buenos aires', 'essec  essec business school', 'fordham university', 'wellesley college', 'skidmore college', 'franklin  marshall college', 'university of massachusetts at amherst  isenberg school of management', 'suffolk university law school', 'university of california berkeley_university of california berkeley', 'uc berkeley', 'university of southern california_university of southern california', 'williams college', 'colby college', 'drew university', 'university of rochester  william e simon graduate school of business administration', 'denison university', 'lafayette college', 'grinnell college', 'calvin college', 'bridgewater state college', 'stonehill college', 'city university of new yorkqueens college', 'hamilton college', 'renmin university of china', 'harbin institute of technology', 'national taiwan university', 'lehigh university  college of business and economics', 'illinois wesleyan university', 'merrimack college', 'wharton school university of pennsylvania', 'tsinghua university_tsinghua university', 'tulane university  ab freeman school of business']\n",
      "\n",
      "\n",
      "['the american university in cairo', 'instituto tecnológico y de estudios superiores de monterrey', 'the university of british columbia  ubc', 'université catholique de louvain', 'university of alberta_university of alberta', 'tongji university', 'the university of hong kong', 'university of british columbia', 'univerzita karlova v praze', 'concordia university_concordia university', 'the hong kong university of science and technology', 'dawson college', 'université de montréal  ecole polytechnique de montréal', 'university of tehran', 'acadia university', 'université paris dauphine', 'british columbia institute of technology  bcit', 'sharif university of technology', 'université du québec à montréal', 'université de montréal  hec montréal', 'project management institute pmi', 'east china normal university', 'bishops university', 'mcgill university_mcgill university']\n",
      "\n",
      "\n",
      "['monash university', 'university of technology sydney', 'university of southern queensland', 'tafe']\n",
      "\n",
      "\n",
      "['ryerson university', 'the university of british columbia', 'mcgill university', 'the university of western ontario', 'university of toronto  rotman school of management']\n",
      "\n",
      "\n",
      "['texas tech university', 'old dominion university', 'the university of akron', 'the university of texas at dallas', 'university of houston', 'kansas state university', 'university of louisiana at lafayette', 'case western reserve university  weatherhead school of management', 'clemson university', 'the university of new mexico', 'university of virginia  darden graduate school of business administration', 'the ohio state university  the max m fisher college of business', 'university of alabama at birmingham', 'north carolina state university_north carolina state university', 'state university of new york at stony brook', 'new jersey institute of technology', 'oklahoma state university', 'university of toledo', 'the ohio state university_the ohio state university', 'wright state university', 'yale university  yale school of management', 'university of kentucky', 'national institute of technology karnataka', 'louisiana state university', 'university of iowa  henry b tippie college of business', 'michigan state university  the eli broad graduate school of management']\n",
      "\n",
      "\n",
      "['insead', 'london business school', 'cfa institute']\n",
      "\n",
      "\n",
      "['university of toronto', 'queens university']\n",
      "\n",
      "\n",
      "['harvard university', 'princeton university', 'brown university', 'columbia university in the city of new york', 'american university', 'the university of chicago  booth school of business', 'harvard law school', 'georgetown university', 'shanghai jiao tong university', 'tsinghua university']\n",
      "\n",
      "\n",
      "['singapore polytechnic', 'university of wollongong', 'victoria university', 'institute of chartered accountants', 'mt eliza business school', 'university of canterbury', 'murdoch university', 'flinders university', 'institute of internal auditors', 'insead sg', 'university of melbourne_university of melbourne']\n",
      "\n",
      "\n",
      "['macomb community college', 'university of wisconsinstout', 'university of wisconsinriver falls', 'the university of north dakota', 'san jose state university_san jose state university', 'university of wisconsinwhitewater', 'seattle pacific university', 'valencia community college', 'city university of new yorkbrooklyn college', 'brigham young university  idaho', 'université de liège', 'ball state university  college of business', 'university of saint thomas', 'butler university', 'minnesota state university mankato', 'wittenberg university', 'georgia southern university', 'university of wisconsinparkside', 'barilan university', 'milwaukee school of engineering', 'state university of new york college at geneseo', 'columbia college', 'tarrant county college', 'st johns university', 'columbus college of art  design', 'colorado state university_colorado state university', 'rosehulman institute of technology', 'bethel university', 'rutgers the state university of new jerseynew brunswick_rutgers the state university of new jerseynew brunswick', 'central missouri state university', 'monmouth university', 'thapar institute of engineering and technology', 'penn state university_penn state university', 'university of central oklahoma', 'sam houston state university', 'university of portland', 'ramapo college of new jersey', 'east tennessee state university', 'brigham young university_brigham young university', 'abilene christian university', 'ivy tech community college', 'university of st thomas', 'university of wisconsinplatteville', 'northwest missouri state university', 'harper college', 'university of north carolina at wilmington', 'mercy college', 'oakland community college', 'nassau community college', 'university of north florida', 'dickinson college', 'university of houstondowntown', 'state university of new york college at plattsburgh', 'plymouth state university', 'strayer university va', 'western carolina university']\n",
      "\n",
      "\n",
      "['the university of south dakota', 'utah valley university', 'winona state university', 'arkansas state university', 'indiana university bloomington_indiana university bloomington', 'damascus university', 'lakeland college', 'university of wisconsingreen bay', 'otis college of art and design', 'univerzitet sv kiril i metódij vo skopje', 'de la salle university', 'devry institute of technology', 'vancouver film school', 'kutztown university of pennsylvania', 'pacific lutheran university', 'davenport university', 'indiana state university', 'south dakota state university', 'belmont university', 'winthrop university']\n",
      "\n",
      "\n",
      "['university of gloucestershire', 'university of the west indies st augustine campus', 'st marys college', 'university of wales', 'panepistimion patron', 'university college cork_university college cork', 'university of bedfordshire', 'national college of ireland', 'university of south africauniversiteit van suidafrika', 'sheffield university', 'manchester metropolitan university', 'aristoteleion panepistimion thessalonikis', 'farnborough college of technology', 'hogeschool inholland', 'sofia university st kliment ohridski', 'the university of kent', 'glasgow caledonian university', 'university of moratuwa', 'university of southampton_university of southampton', 'university of northumbria at newcastle', 'southampton university', 'universiteit leiden', 'nottingham trent university', 'durham university']\n",
      "\n",
      "\n",
      "['university of leeds', 'kings college london u of london', 'university of warwick', 'university college london u of london', 'imperial college london', 'the university of edinburgh', 'university of bath', 'university of london', 'university of york']\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print \"institution\" clusters\n",
    "n_clusters = n_cluster_5\n",
    "cluster = AgglomerativeClustering(n_clusters=n_clusters, affinity='euclidean', linkage='ward') \n",
    "\n",
    "model5vec_list = list()\n",
    "for i in list(model5.wv.vocab.keys()):\n",
    "    model5vec_list.append(model5[i])\n",
    "model5vec_cluster = list(cluster.fit_predict(model5vec_list))\n",
    "\n",
    "model5vec = dict()\n",
    "for i in range(len(model5.wv.vocab.keys())):\n",
    "    model5vec[list(model5.wv.vocab.keys())[i]] = model5vec_cluster[i]\n",
    "print('\\n')\n",
    "model5_cluster_list = list()\n",
    "for i in range(0,n_clusters):\n",
    "    print([k for k,v in model5vec.items() if v == i])\n",
    "    model5_cluster_list.append([k for k,v in model5vec.items() if v == i])\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "['research', 'oil', 'energy', 'human', 'resources', 'broadcast', 'media', 'food', 'beverages', 'design', 'computer', 'online', 'banking', 'services', 'management', 'financial', 'goods', 'real', 'estate', 'medical', 'devices', 'building', 'materials', 'mining', 'metals', 'higher', 'law', 'practice', 'production', 'market', 'information', 'electronics', 'graphic', 'executive', 'office', 'apparel', 'fashion', 'leisure', 'travel', 'tourism', 'defense', 'space', 'public', 'policy', 'government', 'administration', 'aviation', 'aerospace', 'events', 'relations', 'architecture', 'planning', 'legal', 'environmental', 'safety', 'enforcement', 'facilities', 'political', 'civic_social', 'commercial', 'financiers', 'individual_family', 'glass_ceramics', 'concrete', 'paper_forest', 'products']\n",
      "\n",
      "\n",
      "['venture', 'equity', 'markets']\n",
      "\n",
      "\n",
      "['mechanical', 'industrial', 'engineering', 'automation', 'civil']\n",
      "\n",
      "\n",
      "['capital', 'private']\n",
      "\n",
      "\n",
      "['professional', 'training', 'coaching']\n",
      "\n",
      "\n",
      "['internet', 'pharmaceutical', 'restaurant', 'telecommunication', 'tobacco', 'insurance', 'hospitality', 'construction', 'semiconductor', 'wireless', 'entertainment', 'printing', 'accounting', 'automotive', 'biotechnology', 'think_tanks', 'retail', 'nanotechnology', 'animation', 'utility', 'chemical', 'cosmetic', 'publishing', 'program_development', 'fine_art', 'sport', 'machinery', 'plastic', 'performing_arts', 'music', 'library', 'gambling_casinos', 'furniture', 'photography', 'maritime', 'textile', 'auto', 'newspaper', 'farming', 'dairy', 'religious_institutions', 'fundraising', 'international_affairs', 'military', 'wholesale', 'philanthropy', 'veterinary', 'shipbuilding']\n",
      "\n",
      "\n",
      "['consumer', 'sporting', 'luxury', 'jewelry']\n",
      "\n",
      "\n",
      "['networking', 'software', 'games', 'hardware', 'network', 'security']\n",
      "\n",
      "\n",
      "['hospital', 'health', 'care', 'mental']\n",
      "\n",
      "\n",
      "['education', 'consulting', 'investment', 'nonprofit', 'organization']\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print \"industry\" clusters\n",
    "n_clusters = n_cluster_6\n",
    "cluster = AgglomerativeClustering(n_clusters=n_clusters, affinity='euclidean', linkage='ward') \n",
    "\n",
    "model6vec_list = list()\n",
    "for i in list(model6.wv.vocab.keys()):\n",
    "    model6vec_list.append(model6[i])\n",
    "model6vec_cluster = list(cluster.fit_predict(model6vec_list))\n",
    "\n",
    "model6vec = dict()\n",
    "for i in range(len(model6.wv.vocab.keys())):\n",
    "    model6vec[list(model6.wv.vocab.keys())[i]] = model6vec_cluster[i]\n",
    "print('\\n')\n",
    "model6_cluster_list = list()\n",
    "for i in range(0,n_clusters):\n",
    "    print([k for k,v in model6vec.items() if v == i])\n",
    "    model6_cluster_list.append([k for k,v in model6vec.items() if v == i])\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "['best buy', 'target', 'target_target', 'best buy_best buy', 'jcpenney_jcpenney', 'jcpenney']\n",
      "\n",
      "\n",
      "['nike_nike', 'dell', 'hewlettpackard', 'nike', 'dell_dell', 'hewlettpackard_hewlettpackard', 'eastman kodak', 'eastman kodak_eastman kodak']\n",
      "\n",
      "\n",
      "['ernst  young_ernst  young', 'ernst  young', 'kpmg', 'barclays bank_barclays bank', 'pwc', 'lloyds tsb', 'axa', 'royal bank of scotland', 'pwc_pwc', 'macquarie bank', 'barclays bank', 'lloyds banking group_lloyds banking group', 'kpmg_kpmg', 'westpac', 'royal bank of scotland_royal bank of scotland', 'commonwealth bank of australia', 'suncorp', 'amp', 'lloyds banking group', 'nationwide building society', 'commonwealth bank of australia_commonwealth bank of australia']\n",
      "\n",
      "\n",
      "['microsoft_microsoft', 'microsoft corporation', 'microsoft', 'yahoo', 'carnegie mellon university', 'linkedin_linkedin', 'yahoo_yahoo', 'linkedin', 'amazoncom', 'ebay', 'microsoft corporation_microsoft corporation', 'amazoncom_amazoncom', 'ebay_ebay']\n",
      "\n",
      "\n",
      "['wells fargo home mortgage', 'countrywide home loans', 'blackrock', 'ge capital_ge capital', 'ge capital', 'us bank', 'wells fargo bank', 'hewitt associates', 'citigroup_citigroup', 'citibank_citibank', 'suntrust bank', 'bny mellon', 'jefferies  company', 'ge', 'watson wyatt', 'capital one', 'pwc consulting', 'icici bank', 'credit suisse first boston', 'the bank of new york', 'northern trust', 'merrill lynch_merrill lynch', 'state street', 'sungard', 'citi group', 'charles schwab', 'banc of america securities', 'principal financial group', 'state street bank', 'bloomberg lp', 'towers watson', 'banco santander', 'national city bank', 'metlife', 'prudential financial', 'the hartford', 'thomson financial', 'ameriprise financial', 'citibank na', 'fifth third bank', 'bankers trust', 'macquarie group', 'andersen consulting', 'reuters', 'ameriprise financial services inc', 'ing', 'wachovia a wells fargo company', 'morgan stanley smith barney', 'bank of new york mellon', 'deutsche bank securities', 'capital one_capital one', 'aig', 'standard  poors', 'zurich financial services', 'new york life insurance company', 'circuit city', 'franklin templeton investments', 'tiaacref', 'wachovia bank', 'wachovia', 'putnam investments', 'nyse euronext', 'columbia business school', 'ge_ge', 'pioneer investments', 'cibc world markets', 'allen  overy', 'barclays wealth', 'rb', 'barclays bank plc', 'bank of ireland', 'pricewaterhousecoopers_pricewaterhousecoopers', 'standard bank', 'rbs markets  international banking', 'bank of china']\n",
      "\n",
      "\n",
      "['pfizer', 'novartis', 'astrazeneca', 'johnson  johnson', 'abbott laboratories', 'abbott laboratories_abbott laboratories', 'pfizer_pfizer', 'novartis_novartis', 'eli lilly', 'glaxosmithkline_glaxosmithkline', 'quintiles', 'glaxosmithkline', 'amgen', 'sanofiaventis', 'pfizer pharmaceuticals', 'astrazeneca_astrazeneca', 'eli lilly and company', 'johnson  johnson_johnson  johnson', 'wyeth pharmaceuticals', 'sanofiaventis_sanofiaventis', 'bristolmyers squibb_bristolmyers squibb', 'bristolmyers squibb', 'merck', 'baxter healthcare_baxter healthcare', 'amgen_amgen', 'scheringplough', 'baxter healthcare', 'merck_merck', 'genentech', 'boehringer ingelheim', 'eli lilly and company_eli lilly and company']\n",
      "\n",
      "\n",
      "['mtv networks', 'verizon wireless', 'freelance', 'electronic arts', 'digitas', 'selfemployed', 'att', 'aol', 'the walt disney company', 'tmobile', 'verizon wireless_verizon wireless', 'att_att', 'electronic arts_electronic arts', 'various companies', 'northrop grumman', 'aol_aol']\n",
      "\n",
      "\n",
      "['cisco', 'motorola mobile devices', 'alcatellucent', 'nortel', 'texas instruments', 'apple', 'nvidia', 'freescale semiconductor', 'intel', 'huawei', 'qualcomm', 'apple inc', 'ibm research', 'nortel networks', 'mentor graphics', 'national semiconductor', 'amd', 'apple inc_apple inc', 'broadcom', 'intel_intel', 'xilinx', 'texas instruments_texas instruments', 'lucent technologies', 'lucent technologies_lucent technologies', 'samsung', 'cypress semiconductor', 'analog devices', 'synopsys', 'motorola mobility', 'cadence design systems', 'autodesk', 'apple computer', 'amd_amd', 'alcatellucent_alcatellucent']\n",
      "\n",
      "\n",
      "['shell', 'technip', 'total ep', 'fluor', 'worleyparsons', 'talisman energy', 'imperial college london', 'exxonmobil_exxonmobil', 'exxonmobil', 'weatherford', 'ghd', 'shell_shell', 'saudi aramco_saudi aramco', 'wood group', 'petrofac', 'amec', 'bhp billiton', 'saudi aramco', 'rio tinto', 'snclavalin', 'subsea 7', 'conocophillips', 'bg group', 'atkins', 'ch2m hill', 'conocophillips_conocophillips', 'santos ltd', 'jacobs engineering', 'woodside energy', 'bhp billiton_bhp billiton']\n",
      "\n",
      "\n",
      "['cgi', 'amazon', 'ibm', 'oracle_oracle', 'mit media lab', 'oracle', 'adobe systems', 'oracle corporation', 'expedia', 'symantec', 'hp', 'intuit', 'paypal', 'sun microsystems', 'vmware', 'independent', 'stony brook university', 'netapp', 'sap', 'san jose state university', 'salesforcecom', 'university of southern california', 'siebel systems', 'intuit_intuit', 'salesforcecom_salesforcecom', 'yahoo inc', 'emc', 'zynga', 'informatica', 'princeton university', 'adobe', 'sun microsystems_sun microsystems', 'klatencor', 'oracle corporation_oracle corporation', 'symantec_symantec', 'twitter', 'novell', 'digital equipment corporation', 'sap_sap', 'uc san diego', 'microsoft research asia']\n",
      "\n",
      "\n",
      "['kelly services', 'discovery communications', 'examinercom', 'pitney bowes', 'clear channel radio', 'computer sciences corporation', 'rutgers university', 'university of oxford', 'groupon', 'aflac', 'us army corps of engineers', 'independent consultant', 'turner construction', 'apple retail', 'mit sloan school of management', 'corporate executive board', 'paramount pictures', 'slalom consulting', 'cerner corporation', 'reliance communications', 'at kearney', 'img', 'stanford graduate school of business', 'cintas', 'freelancer', 'indian army', 'lawrence berkeley national laboratory', 'gameloft', 'self employed', 'right management', 'bearingpoint', 'mckesson', 'patni computer systems', 'live nation', 'army national guard', 'express scripts', 'unitedhealth group', 'university of rochester', 'na', 'frost  sullivan', 'university of melbourne', 'urban outfitters', 'united states senate', 'usaf', 'sapientnitro', 'gensler', 'logica', 'cap gemini ernst  young', 'towers perrin', 'sony music entertainment', 'razorfish', 'safeway', 'united nations', 'university of phoenix', 'university college dublin', 'various', 'academy of art university', 'ncr', 'nbc universal', 'clemson university', 'monster', 'honeywell aerospace', 'globallogic', 'usaid', 'telecom new zealand', 'independant consultant', 'israel defense forces', 'general dynamics', 'siemens', 'dominos pizza', 'scholastic', 'edelman', 'hult international business school', 'london business school', 'university of north texas', 'accenture technology solutions', 'sutherland global services', 'san diego state university', 'washington state university', 'university of queensland', 'university of chicago', 'rochester institute of technology', 'zs associates', 'colliers international', 'mary kay inc', 'syntel', 'mayo clinic', 'ptc', 'iron mountain', 'thales', 'teach for america', 'american cancer society', 'confidential', 'cbs interactive', 'xerox corporation', 'mtn', 'us house of representatives', 'nbcuniversal inc', 'mckesson provider technologies', 'concordia university', 'luxoft', 'florida state university', 'university of cambridge', 'sodexo', 'universal music group', 'gamestop', 'universal studios', 'unicef', 'origin energy', 'target corporation', 'caesars entertainment corporation', 'university of limerick', 'itv', 'fairfax media', 'cbeyond', 'home', 'bloomingdales', 'oxford university press', 'vanderbilt university', 'university of new south wales', 'devry university', 'allstate', 'the university of arizona', 'university of arizona', 'none', 'telecom new zealand_telecom new zealand', 'turner broadcasting', 'freelance_freelance', 'huron consulting group', 'jcrew', 'moss adams llp', 'ef education first', 'northrop grumman corporation', 'syracuse university', 'robert bosch', 'sainsburys', 'manpower inc', 'future group', 'queen mary university of london', 'humber college', 'national university of singapore', 'idea cellular ltd', 'university of georgia', 'fmc technologies', 'the world bank', 'bskyb', 'cr bard', 'bbc worldwide', 'tc', 'monsanto', 'georgiapacific llc', 'sams club', 'walmart stores inc', 'macquarie university', 'farmers insurance', 'kaplan', 'brigham young university', 'jaguar land rover', 'bc hydro', 'kornferry international', 'robert walters', 'university of california', 'lee hecht harrison', 'european parliament', 'aetna', 'ikon office solutions', 'genpact llc', 'undp', 'orange', 'american family insurance', 'rice university', 'spherion', 'arrow electronics', 'global crossing', 'reed business information', 'ncr corporation', 'coach', 'shaw communications', 'university of new mexico', 'tmp worldwide', 'pearson education', 'cable  wireless', 'private', 'university of miami', 'jones day', 'ymca', 'university of texas at arlington', 'thoughtworks', 'trend micro', 'agilent technologies', 'university of oregon', 'temple university', 'nanyang technological university', 'hca', 'booz  company', 'ed', 'us census bureau', 'td canada trust', 'georgia state university', 'american airlines', 'hbo', 'united states army reserve', 'tn', 'oregon state university', 'boy scouts of america', 'massachusetts institute of technology', 'convergys', 'transcanada', 'cnn', 'randstad', 'pa consulting group', 'sony electronics', 'coldwell banker', 'british army', 'deloitte australia', 'kansas state university', 'united states army', 'waste management inc', 'keybank', 'obama for america', 'bt global services', 'rr donnelley', 'network rail', 'electronic data systems', 'vector marketing', 'red hat', 'first data corporation', 'reliant energy', 'balfour beatty construction', 'emory university', 'comcast cable', 'southern california edison', 'black  veatch', 'guess', 'hdr', 'williamssonoma inc', 'walt disney world', 'university of ottawa', 'mattel', 'jacob', 'university of southampton', 'edf energy', 'department of veterans affairs', 'itt technical institute', 'university of central florida', 'hr block', 'cablevision', 'the new york times', 'banana republic', 'manpower', 'tommy hilfiger', 'university of alberta', 'express', 'the dow chemical company', 'westinghouse electric company', 'technicolor', 'university of leeds', 'international rescue committee', 'cranfield university', 'british telecom', 'british army_british army', 'monitor group']\n",
      "\n",
      "\n",
      "['lowes home improvement', 'washington mutual', 'thomson reuters', 'wells fargo', 'officemax', 'northwestern mutual', 'robert half international', 'ceridian', 'usmc', 'mci', 'sovereign bank', 'gap inc', 'paychex', 'att mobility', 'unisys', 'fiserv', 'sears holdings corporation', 'humana', 'espn', 'ups', 'cigna', 'traveler', 'comcast', 'adp_adp', 'adp', 'time inc', 'united states navy', 'sprint', 'western union', 'cleveland clinic', 'liberty mutual', 'pnc', 'staple', 'abercrombie  fitch', 'arthur andersen  co', 'aon hewitt', 'ecolab', 'comerica bank', 'nordstrom', 'portland state university', 'walgreens', 'bank one', 'qwest communications', 'sherwin williams', 'bni', 'att wireless', 'anthropologie', 'liberty mutual insurance', 'usaa', 'limited brands', 'massmutual financial group', 'harris corporation', 'wells fargo financial', 'enterprise rentacar', 'verizon', 'macys', 'george washington university', 'compusa', 'freddie mac', 'dicks sporting goods', 'sprint nextel', 'sears holdings corporation_sears holdings corporation', 'wells fargo_wells fargo', 'teksystems', 'cox communications', 'saks fifth avenue', 'walgreens_walgreens', 'united airlines', 'american eagle outfitters', 'pizza hut', 'thomson reuters_thomson reuters', 't rowe price', 'depaul university', 'kohls department stores', 'blockbuster', 'verizon business', 'xo communications', 'may department stores', 'bbt', 'old navy', 'victorias secret', 'commerce bank']\n",
      "\n",
      "\n",
      "['accenture_accenture', 'accenture', 'cognizant technology solutions', 'deloitte', 'deloitte consulting', 'tata consultancy services', 'infosys', 'wipro technologies', 'vodafone', 'telstra', 'tata consultancy services_tata consultancy services', 'visa', 'infosys technologies ltd', 'capgemini', 'sapient', 'cognizant technology solutions_cognizant technology solutions', 'telstra_telstra', 'vodafone_vodafone']\n",
      "\n",
      "\n",
      "['united states marine corps', 'north carolina state university', 'shell oil company', 'bose corporation', 'uk ministry of defence', 'parson', 'ingersoll rand', 'carrier corporation', 'university of houston', 'texas am university', 'honeywell process solutions', 'petroleum development oman', 'aker solutions', 'university of calgary', 'kbr', 'mott macdonald', 'national oilwell varco', 'urs corporation', 'foster wheeler', 'abb', 'kaiser permanente', 'university of kansas', 'alstom power', 'emerson process management', 'aecom', 'halcrow', 'suncor energy', 'hatch', 'erm', 'oxy', 'hess corporation', 'mustang engineering', 'devon energy', 'bechtel', 'encana', 'cameron', 'sinclair knight merz', 'royal australian navy', 'john holland', 'bombardier transportation', 'kbr inc', 'aecom australia pty ltd', 'bechtel corporation', 'leighton contractors']\n",
      "\n",
      "\n",
      "['allergan', 'roche diagnostics', 'roche pharmaceuticals', 'honeywell', 'university of delaware', 'cvs caremark corporation', 'stryker', 'cardinal health', 'quest diagnostics', 'bayer healthcare', 'university of british columbia', 'cipla', 'siemens healthcare', 'philip', 'university of north carolina at chapel hill', 'novartis pharmaceuticals', 'drexel university', 'takeda pharmaceuticals', 'wyeth', 'black  decker', 'massachusetts general hospital', 'bayer', 'philips healthcare', 'st jude medical', 'boston scientific', 'teva pharmaceuticals', 'roche', 'genzyme', 'medtronic', 'cvs pharmacy', 'boston scientific_boston scientific', 'smith  nephew', 'philips medical systems', 'harvard medical school', 'merck sharp  dohme', 'gsk', 'biogen idec', 'reckitt benckiser', 'sandoz', 'shire pharmaceuticals', 'parexel', 'university of victoria', 'gilead sciences', 'boehringer ingelheim pharmaceuticals', 'philip_philip', 'medtronic_medtronic', 'janssen pharmaceutical companies of johnson and johnson', 'ppd', 'national institutes of health']\n",
      "\n",
      "\n",
      "['new york university', 'columbia university', 'virginia tech', 'university of colorado at boulder', 'nasa ames research center', 'ieee', 'georgia institute of technology', 'stanford university', 'iowa state university', 'sri international', 'university of michigan', 'mit', 'indiana university', 'carnegie mellon university_carnegie mellon university', 'arizona state university', 'sandia national laboratories', 'northeastern university', 'university of florida', 'university of texas at austin', 'idf', 'university of illinois at urbanachampaign', 'purdue university', 'the university of texas at austin', 'harvard university', 'university of maryland college park', 'university of california irvine', 'university of washington', 'dartmouth college', 'university of california berkeley', 'mit lincoln laboratory', 'inria', 'hewlettpackard laboratories', 'ucla', 'the ohio state university', 'johns hopkins university', 'compuware', 'applied materials', 'university of pennsylvania', 'nuance communications', 'northwestern university', 'cornell university', 'los alamos national laboratory', 'boston university', 'colorado state university', 'uc davis', 'indian institute of science', 'university of wisconsinmadison', 'duke university', 'university of virginia', 'microsoft research', 'california institute of technology', 'university of maryland', 'university of massachusetts amherst', 'argonne national laboratory', 'yale university', 'university of minnesota', 'university of illinois at urbanachampaign_university of illinois at urbanachampaign', 'us department of state', 'brown university', 'tufts university', 'florida international university', 'lawrence livermore national laboratory']\n",
      "\n",
      "\n",
      "['procter  gamble_procter  gamble', 'procter  gamble']\n",
      "\n",
      "\n",
      "['the home depot', 'mcdonalds corporation', 'university of cincinnati', 'hay', 'starbucks', 'marriott international', 'nielsen', 'university of sydney', 'john deere', 'sears canada', 'ranbaxy', 'bureau veritas', 'davita', 'leo burnett', 'larsen  toubro limited', 'shoppers drug mart', 'dr reddys laboratories', 'university of surrey', 'kings college london', 'hudson', 'coopers  lybrand', 'aerotek', 'rbc dominion securities', 'university of birmingham', 'mcdonalds', 'toys r us', 'golder associates', 'kpmg peat marwick', 'walmart', '3m_3m', 'canadian tire', 'ec harris', 'university of bristol', 'cb richard ellis', 'vanguard_vanguard', 'epam systems', 'oklahoma state university', 'four seasons hotels and resorts', 'thermo fisher scientific', 'university of manchester', 'willis', 'eversheds', 'michael page', 'intercontinental hotels group', 'european commission', 'debenhams', 'rmit university', 'starbucks coffee company', 'american heart association', 'loblaw companies limited', 'the home depot_the home depot', 'financial times', 'aon', 'hay group', 'csiro', 'experian', 'hallmark cards', 'tesco', 'delta air lines', 'national bank financial', 'fairmont hotels and resorts', 'johnson controls automotive', 'toyota', 'dhl', 'deloitte tax llp', 'sears roebuck and co', 'private practice', 'bbc_bbc', 'starwood hotels  resorts worldwide inc', '3m', 'nationwide insurance', 'diageo', 'cargill', 'international paper', 'aramark', 'washington university in st louis', 'rio tinto_rio tinto', 'marcus  millichap', 'national grid', 'harrod', 'levi strauss  co', 'the body shop', 'boot', 'loreal', 'hyatt hotels corporation', 'john lewis', 'marks and spencer', 'keller williams realty', 'disney consumer products', 'key bank', 'bdo llp', 'beckman coulter_beckman coulter', 'sg', 'grant thornton uk llp', 'basf', 'starwood hotels  resorts', 'university college london', 'parsons brinckerhoff', 'tesco stores ltd', 'ge money', 'lend lease', 'marriott international_marriott international', 'intercontinental hotels group_intercontinental hotels group', 'regus', 'the cooperative group', 'dow chemical', 'oliver wyman', 'conagra foods', 'ims health', 'price waterhouse', 'monash university', 'jwt', 'ghent university', 'bbc', 'turner  townsend', 'newmont mining corporation', 'bausch  lomb', 'hilton worldwide', 'cardiff university', 'arthur d little', 'beckman coulter', 'newell rubbermaid', 'woolworth', 'barrick gold corporation', 'ikea', 'woolworths limited', 'capita symonds', 'sara lee', 'avery dennison', 'laing orourke', 'carillion', 'nh', 'burberry', 'rsm tenon', 'cocacola enterprises', 'loréal', 'allstate insurance', 'university of nottingham', 'polo ralph lauren', 'brisbane city council', 'sony computer entertainment europe', 'allianz', 'walmart_walmart', 'tesco_tesco', 'savills', 'rio tinto alcan']\n",
      "\n",
      "\n",
      "['the boeing company', 'case western reserve university', 'general electric', 'boeing', 'peace corps', 'independent contractor', 'university of pittsburgh', 'covidien', 'state farm insurance', 'xerox', 'gerson lehrman group', 'eaton corporation', 'general electric_general electric', 'progressive insurance', 'johnson controls', 'rollsroyce', 'lear corporation', 'american red cross', 'delphi_delphi', 'delphi', 'harleydavidson motor company', 'bombardier aerospace', 'cummins inc', 'owens corning', 'caterpillar inc_caterpillar inc', 'caterpillar inc', 'chrysler', 'michigan state university', 'rockwell automation', 'sears', 'trw', 'bombardier aerospace_bombardier aerospace', 'xerox_xerox', 'rockwell automation_rockwell automation', 'daimlerchrysler', 'nasa', 'alcoa', 'state farm insurance_state farm insurance', 'pratt  whitney', 'university of connecticut', 'georgetown university', 'george mason university', 'airbus', 'delphi automotive systems', 'johnson controls_johnson controls']\n",
      "\n",
      "\n",
      "['philip morris international', 'dupont', 'pepsico_pepsico', 'reliance industries limited', 'pepsico', 'hindustan unilever ltd', 'general mills_general mills', 'arup', 'ppg industries', 'british american tobacco', 'mar', 'hj heinz', 'kimberlyclark_kimberlyclark', 'british american tobacco_british american tobacco', 'the cocacola company', 'kimberlyclark', 'the cocacola company_the cocacola company', 'nestle', 'toastmasters international', 'the nielsen company', 'kellogg company_kellogg company', 'kellogg company', 'colgate palmolive', 'sc johnson', 'goodman fielder', 'campbell soup company', 'united biscuits', 'general mills', 'fonterra', 'frito lay', 'the nielsen company_the nielsen company', 'national grid_national grid', 'nestle_nestle', 'colgate palmolive_colgate palmolive']\n",
      "\n",
      "\n",
      "['fidelity investments', 'fidelity investments_fidelity investments']\n",
      "\n",
      "\n",
      "['morgan stanley', 'jp morgan', 'citibank', 'goldman sachs_goldman sachs', 'goldman sachs', 'lehman brothers', 'citi', 'citigroup', 'societe generale', 'bank of america merrill lynch', 'ubs investment bank_ubs investment bank', 'financial services authority', 'ubs investment bank', 'credit suisse_credit suisse', 'jp morgan_jp morgan', 'merrill lynch', 'credit suisse', 'deutsche bank_deutsche bank', 'jp morgan chase', 'nomura', 'morgan stanley_morgan stanley', 'bnp paribas', 'deutsche bank', 'barclays capital', 'bear stearns', 'bloomberg', 'jpmorgan chase', 'ubs', 'ubs wealth management', 'jpmorgan chase_jpmorgan chase', 'citi_citi', 'dbs bank', 'barclays', 'jpmorgan', 'abn amro', 'barclays capital_barclays capital', 'commerzbank ag', 'rbs global banking  markets', 'abn amro bank nv']\n",
      "\n",
      "\n",
      "['us navy', 'ge energy_ge energy', 'us navy_us navy', 'ge energy']\n",
      "\n",
      "\n",
      "['cisco systems', 'bt_bt', 'bt', 'cisco systems_cisco systems']\n",
      "\n",
      "\n",
      "['ibm canada ltd', 'university of toronto', 'samsung electronics', 'sony', 'ryerson university', 'university of edinburgh', 'bell canada', 'ubisoft', 'rogers communications inc', 'telus', 'trinity college dublin', 'university of waterloo', 'queens university', 'the university of british columbia', 'york university', 'mcmaster university', 'sony pictures entertainment', 'ideo', 'university of guelph', 'reliance retail', 'mcgill university', 'university of toronto_university of toronto', 'itc limited', 'schneider electric', 'lg electronics', 'carleton university', 'simon fraser university', 'carphone warehouse', 'self', 'life technologies', 'sybase', 'nokia mobile phones', 'tmobile_tmobile', 'rogers communications', 'eircom', 'canadian forces']\n",
      "\n",
      "\n",
      "['american express', 'american express_american express']\n",
      "\n",
      "\n",
      "['weatherford_weatherford', 'halliburton', 'schlumberger', 'baker hughes', 'chevron', 'halliburton_halliburton', 'schlumberger_schlumberger', 'chevron_chevron', 'baker hughes_baker hughes']\n",
      "\n",
      "\n",
      "['ford motor company', 'ford motor company_ford motor company']\n",
      "\n",
      "\n",
      "['direct energy', 'deloitte_deloitte', 'aon consulting', 'jones lang lasalle', 'mercer human resource consulting', 'axis bank', 'bdo usa llp', 'kpmg llp', 'arthur andersen', 'protiviti', 'td', 'fannie mae', 'serco', 'united bank limited', 'deloitte consulting_deloitte consulting', 'genpact', 'rbc royal bank', 'evalueserve', 'pricewaterhousecoopers', 'rbc', 'aviva plc', 'legal  general', 'manulife financial', 'bank of montreal', 'deloitte  touche llp', 'ogilvy  mather', 'kpmg advisory', 'scotiabank', 'grant thornton', 'british council', 'prudential insurance', 'virtusa', 'tata steel', 'price waterhouse coopers', 'epic', 'deloitte  touche', 'smith barney', 'tata motors', 'rbc capital markets', 'mercer', 'cibc', 'union bank', 'sun life financial', 'office depot', 'royal bank of canada', 'td securities', 'cushman  wakefield', 'cantor fitzgerald', 'ernst and young', 'adecco', 'consultant', 'baker  mckenzie', 'rga', 'british gas', 'ernst  young llp', 'yes bank', 'mt bank', 'hudsons bay company', 'arthur andersen llp', 'capgemini consulting', 'grant thornton llp', 'lazard', 'kpmg audit', 'synovate', 'resources global professionals', 'national bank of canada', 'kpmg us', 'reed', 'caterpillar', 'university of technology sydney', 'heidrick  struggles', 'singapore armed forces', 'hdfc bank', 'cbre', 'marsh', 'dla piper', 'dun  bradstreet', 'caput', 'pricewaterhousecoopers llp', 'swiss re', 'burger king corporation', 'queens university belfast', 'wellpoint', 'avon', 'russell investments', 'hbos', 'london 2012', 'linklaters', 'bupa', 'ace group', 'british airways', 'axa advisors', 'optus', 'td bank', 'barclaycard', 'hm forces', 'jones lang lasalle_jones lang lasalle', 'bmo capital markets', 'td bank group', 'edward jones', 'centrica', 'standard life', 'prudential assurance', 'conde nast']\n",
      "\n",
      "\n",
      "['ibm_ibm']\n",
      "\n",
      "\n",
      "['unilever', 'unilever_unilever']\n",
      "\n",
      "\n",
      "['royal air force', 'royal navy_royal navy', 'royal navy', 'royal air force_royal air force']\n",
      "\n",
      "\n",
      "['bank of america', 'bank of america_bank of america']\n",
      "\n",
      "\n",
      "['research in motion', 'research in motion_research in motion']\n",
      "\n",
      "\n",
      "['general motors_general motors', 'general motors']\n",
      "\n",
      "\n",
      "['whirlpool corporation_whirlpool corporation', 'whirlpool corporation', 'chrysler_chrysler']\n",
      "\n",
      "\n",
      "['us army', 'us army_us army']\n",
      "\n",
      "\n",
      "['empowered networkers inc_empowered networkers inc']\n",
      "\n",
      "\n",
      "['facebook', 'google', 'google inc', 'stanford university_stanford university', 'uc berkeley', 'intel corporation_intel corporation', 'google_google', 'intel corporation']\n",
      "\n",
      "\n",
      "['nokia siemens networks_nokia siemens networks', 'nokia siemens networks', 'nokia', 'motorola', 'nokia_nokia', 'motorola_motorola', 'nortel_nortel']\n",
      "\n",
      "\n",
      "['standard chartered bank_standard chartered bank', 'hsbc_hsbc', 'hsbc', 'standard chartered bank']\n",
      "\n",
      "\n",
      "['bp_bp', 'bp']\n",
      "\n",
      "\n",
      "['ericsson', 'ericsson_ericsson']\n",
      "\n",
      "\n",
      "['aiesec', 'the boston consulting group', 'mckinsey  company', 'mckinsey  company_mckinsey  company', 'bain  company', 'harvard business school']\n",
      "\n",
      "\n",
      "['dimension data', 'national instruments', 'advanced micro devices', 'rensselaer polytechnic institute', 'mcafee', 'dell inc', 'sun microsystems inc', 'citrix systems', 'wipro', 'tech mahindra', 'juniper networks', 'sa', 'csc', 'barnes  noble', 'ca technologies', 'avaya', '3com', 'adobe systems_adobe systems', 'motorola solutions', 'university of utah', 'indian institute of technology bombay', 'cisco systems inc', 'mahindra satyam', 'i2 technologies', 'directv', 'stmicroelectronics', 'sandisk', 'lenovo', 'level 3 communications', 'university of illinois at chicago', 'computer associates', 'university of texas at dallas', 'amdocs', 'ibm global business services', 'hewlett packard', 'motorola inc', 'flextronics', 'bharti airtel limited', 'ca', 'seagate technology', 'teradata', 'jet propulsion laboratory', 'fujitsu', 'avid technology', 'persistent systems', 'lexisnexis', 'penn state university', 'hcl technologies', 'infosys_infosys', 'bmc software', 'stanford university school of medicine', 'brocade', 'ibm global services', 'comverse', 'sap labs', 'philips consumer electronics', 'philips semiconductors', 'atos origin', 'verisign', 'time warner cable', 'aspect software', 'alcatel', 'blizzard entertainment', 'illinois institute of technology']\n",
      "\n",
      "\n",
      "['ge healthcare', 'ge healthcare_ge healthcare']\n",
      "\n",
      "\n",
      "['kraft foods_kraft foods', 'kraft foods']\n",
      "\n",
      "\n",
      "['national australia bank_national australia bank', 'national australia bank', 'anz', 'anz_anz']\n",
      "\n",
      "\n",
      "['bae systems', 'bae systems_bae systems', 'united states air force_united states air force']\n",
      "\n",
      "\n",
      "['saic', 'united states air force', 'lockheed martin', 'lockheed martin_lockheed martin', 'us air force', 'l3 communications', 'raytheon', 'booz allen hamilton', 'rockwell collins', 'us air force_us air force']\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print \"company\" clusters\n",
    "n_clusters = n_cluster_7\n",
    "cluster = AgglomerativeClustering(n_clusters=n_clusters, affinity='euclidean', linkage='ward') \n",
    "\n",
    "model7vec_list = list()\n",
    "for i in list(model7.wv.vocab.keys()):\n",
    "    model7vec_list.append(model7[i])\n",
    "model7vec_cluster = list(cluster.fit_predict(model7vec_list))\n",
    "\n",
    "model7vec = dict()\n",
    "for i in range(len(model7.wv.vocab.keys())):\n",
    "    model7vec[list(model7.wv.vocab.keys())[i]] = model7vec_cluster[i]\n",
    "print('\\n')\n",
    "model7_cluster_list = list()\n",
    "for i in range(0,n_clusters):\n",
    "    print([k for k,v in model7vec.items() if v == i])\n",
    "    model7_cluster_list.append([k for k,v in model7vec.items() if v == i])\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Putting feature clusters into dummy variables in numpy array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List of degree types:\n",
      " ['phd', 'bs', 'ba', 'master', 'ms', 'hnd', 'business', 'md', 'management', 'high', 'associate', 'aa', 'btech', 'ab', 'na', 'jd', 'bcom', 'aas', 'beng', 'post_graduate', 'hsc', 'b_sc', 'bcomm', 'bfa', 'engineering', 'ssc', 'meng', 'science', 'pgdm', 'hons', 'b_tech', 'llb', '21', 'llm', 'juris_doctor', 'level', 'b', 'graduate']\n"
     ]
    }
   ],
   "source": [
    "# Skills\n",
    "for i in range(len(model2_cluster_list)):\n",
    "    model2_cluster_list[i] = set(model2_cluster_list[i])\n",
    "\n",
    "# Major\n",
    "for i in range(len(model3_cluster_list)):\n",
    "    model3_cluster_list[i] = set(model3_cluster_list[i])\n",
    "\n",
    "# Degree into the same format\n",
    "model4_cluster_list = list()\n",
    "for i in range(len(model4.wv.vocab.keys())):\n",
    "    model4_cluster_list.append(list(model4.wv.vocab.keys())[i])\n",
    "print('List of degree types:\\n', model4_cluster_list)\n",
    "\n",
    "# Institution\n",
    "for i in range(len(model5_cluster_list)):\n",
    "    model5_cluster_list[i] = set(model5_cluster_list[i])\n",
    "    \n",
    "# Industry\n",
    "for i in range(len(model6_cluster_list)):\n",
    "    model6_cluster_list[i] = set(model6_cluster_list[i])\n",
    "    \n",
    "# Company\n",
    "for i in range(len(model7_cluster_list)):\n",
    "    model7_cluster_list[i] = set(model7_cluster_list[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature array dimensions:  (107632, 219)\n"
     ]
    }
   ],
   "source": [
    "# Work out the dimension of the numpy array and create it\n",
    "array_rows = len(degree)\n",
    "len_2 = len(model2_cluster_list) # skill (50)\n",
    "len_3 = len(model3_cluster_list) # major (20)\n",
    "len_4 = len(model4_cluster_list) # degree (no clustering: 38)\n",
    "len_5 = len(model5_cluster_list) # institution (50)\n",
    "len_6 = len(model6_cluster_list) # industry (10)\n",
    "len_7 = len(model7_cluster_list) # company (50)\n",
    "\n",
    "feature_columns = (len_2 + len_3 + len_4 + len_5 + len_6 + len_7 + 1)\n",
    "feature_array = np.zeros((array_rows, feature_columns))\n",
    "print('Feature array dimensions: ', feature_array.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows and columns for each array:\n",
      "skills: 107632 ,  50\n",
      "major: 107632 ,  20\n",
      "degree: 107632 ,  38\n",
      "institution: 107632 ,  50\n",
      "industry: 107632 ,  10\n",
      "company: 107632 ,  50\n",
      "year_of_work_experience: 107632 ,  1\n",
      "\n",
      "\n",
      "Sample feature row for one profile:\n",
      " [[ 0.  0.  0.  1.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  1.  1.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  1.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "   0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "   0.  0. 16.]]\n",
      "\n",
      "\n",
      "Feature array dimensions:  (107632, 219)\n",
      "\n",
      "\n",
      "Feature array sum:  4786893.0\n"
     ]
    }
   ],
   "source": [
    "print('Number of rows and columns for each array:')\n",
    "\n",
    "# Skills\n",
    "print('skills:', len(skills), ', ', len_2)\n",
    "for person in range(len(skills)):\n",
    "    for item in skills[person]:\n",
    "        for word in item:\n",
    "            for cluster in range(len_2):\n",
    "                if word in model2_cluster_list[cluster]:\n",
    "                    feature_array[person, (cluster)] += 1\n",
    "\n",
    "# Major\n",
    "print('major:', len(major), ', ', len_3)\n",
    "for person in range(len(major)):\n",
    "    for item in major[person]:\n",
    "        for word in item:\n",
    "            for cluster in range(len_3):\n",
    "                if word in model3_cluster_list[cluster]:\n",
    "                    feature_array[person, (len_2 + cluster)] += 1\n",
    "                    \n",
    "# Degree\n",
    "print('degree:', len(degree), ', ', len_4)\n",
    "for person in range(len(degree)):\n",
    "    for item in degree[person]:\n",
    "        for word in item:\n",
    "            for cluster in range(len_4):\n",
    "                if word == model4_cluster_list[cluster]:\n",
    "                    feature_array[person, (len_2 + len_3 + cluster)] += 1\n",
    "                    \n",
    "# Institution\n",
    "print('institution:', len(institution), ', ', len_5)\n",
    "for person in range(len(institution)):\n",
    "    for item in institution[person]:\n",
    "        for word in item:\n",
    "            for cluster in range(len_5):\n",
    "                if word in model5_cluster_list[cluster]:\n",
    "                    feature_array[person, (len_2 + len_3 + len_4 + cluster)] += 1\n",
    "                    \n",
    "# Industry\n",
    "print('industry:', len(industry), ', ', len_6)\n",
    "for person in range(len(industry)):\n",
    "    for item in industry[person]:\n",
    "        for word in item:\n",
    "            for cluster in range(len_6):\n",
    "                if word in model6_cluster_list[cluster]:\n",
    "                    feature_array[person, (len_2 + len_3 + len_4 + len_5 + cluster)] += 1\n",
    "\n",
    "# Company\n",
    "print('company:', len(company), ', ', len_7)\n",
    "for person in range(len(company)):\n",
    "    for item in company[person]:\n",
    "        for word in item:\n",
    "            for cluster in range(len_7):\n",
    "                if word in model7_cluster_list[cluster]:\n",
    "                    feature_array[person, (len_2 + len_3 + len_4 + len_5 + len_6 + cluster)] += 1\n",
    "                    \n",
    "# Years of work experiences\n",
    "year_of_work = df['year_of_work_experience']\n",
    "print('year_of_work_experience:', len(year_of_work), ', ', 1)\n",
    "for person in range(len(year_of_work)):\n",
    "    feature_array[person, (len_2 + len_3 + len_4 + len_5 + len_6 + len_7)] = year_of_work[person]\n",
    "print('\\n')\n",
    "print('Sample feature row for one profile:\\n', feature_array[:1])\n",
    "print('\\n')\n",
    "print('Feature array dimensions: ', feature_array.shape)\n",
    "print('\\n')\n",
    "print('Feature array sum: ', np.sum(feature_array))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Labeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>skills</th>\n",
       "      <th>specilities</th>\n",
       "      <th>interests</th>\n",
       "      <th>institution</th>\n",
       "      <th>degree</th>\n",
       "      <th>major</th>\n",
       "      <th>summary</th>\n",
       "      <th>honors</th>\n",
       "      <th>industry</th>\n",
       "      <th>occupation</th>\n",
       "      <th>company</th>\n",
       "      <th>year_of_work_experience</th>\n",
       "      <th>group</th>\n",
       "      <th>latest occupation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[DNA, Nanotechnology, Molecular Biology, Softw...</td>\n",
       "      <td></td>\n",
       "      <td>personal genomics, nanotechnology</td>\n",
       "      <td>[Harvard University, Yale University]</td>\n",
       "      <td>[PHD, BS]</td>\n",
       "      <td>[Biophysics, Computer Science]</td>\n",
       "      <td>I am interested in inventing new methods to co...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Research</td>\n",
       "      <td>[Assistant Professor, Technology Development F...</td>\n",
       "      <td>[UCSF, Wyss Institute for Biologically Inspire...</td>\n",
       "      <td>16</td>\n",
       "      <td>[]</td>\n",
       "      <td>Assistant Professor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[Interactive Marketing, Content Strategy, Affi...</td>\n",
       "      <td>Internet Marketing, Interactive Marketing, Dig...</td>\n",
       "      <td>Digital Photography, Tennis, Internet Marketin...</td>\n",
       "      <td>[University of Virginia]</td>\n",
       "      <td>[BA]</td>\n",
       "      <td>[History]</td>\n",
       "      <td>Love to work on deadline &amp; data-driven, dynami...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Internet</td>\n",
       "      <td>[Social Media Marketing Manager, Board of Dire...</td>\n",
       "      <td>[Coca-Cola, Atlanta Interactive Marketing Asso...</td>\n",
       "      <td>21</td>\n",
       "      <td>[]</td>\n",
       "      <td>Social Media Marketing Manager</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[Primavera, Revit MEP, AutoCAD, Engineering, H...</td>\n",
       "      <td></td>\n",
       "      <td>Project Management, Innovative Designs, New Te...</td>\n",
       "      <td>[University of Petroleum &amp; Energy Studies, IIT...</td>\n",
       "      <td>[, , , , ]</td>\n",
       "      <td>[, , , , ]</td>\n",
       "      <td>More than 08 yrs of experience with hands on e...</td>\n",
       "      <td>[# Selected in ARMY through SSB (Technical Gra...</td>\n",
       "      <td>Oil &amp; Energy</td>\n",
       "      <td>[Manager (International Business Development &amp;...</td>\n",
       "      <td>[VOITH Hydro Pvt Ltd., VOITH Hydro Pvt. Ltd., ...</td>\n",
       "      <td>22</td>\n",
       "      <td>[]</td>\n",
       "      <td>Manager (International Business Development &amp; ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Talent Acquisition, Recruiting, Talent Manage...</td>\n",
       "      <td></td>\n",
       "      <td>Organizational development, personal developme...</td>\n",
       "      <td>[Universitatea „Transilvania” din Brașov, Univ...</td>\n",
       "      <td>[BA, ]</td>\n",
       "      <td>[Psychology &amp; Science of Education, ]</td>\n",
       "      <td>RecruitmentInternal communicationTrainingNew h...</td>\n",
       "      <td>[]</td>\n",
       "      <td>Human Resources</td>\n",
       "      <td>[Recruitment consultant, Scientific Staffing C...</td>\n",
       "      <td>[CGI, Kelly Services, Carmeuse, Education Inst...</td>\n",
       "      <td>18</td>\n",
       "      <td>[]</td>\n",
       "      <td>Recruitment consultant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[Brand Management, Integrated Marketing, Telev...</td>\n",
       "      <td></td>\n",
       "      <td>Broadcast Design, Television, Creativity, Music.</td>\n",
       "      <td>[Instituto de Diseño de Caracas]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>Accomplished Chief Creative Officer with 15+ y...</td>\n",
       "      <td>[Promax and BDA World Gold Awards, New York Fe...</td>\n",
       "      <td>Broadcast Media</td>\n",
       "      <td>[Creative Services, Vice President, Creative S...</td>\n",
       "      <td>[Discovery Communications, Warner Channel, War...</td>\n",
       "      <td>28</td>\n",
       "      <td>[]</td>\n",
       "      <td>Creative Services</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              skills  \\\n",
       "0  [DNA, Nanotechnology, Molecular Biology, Softw...   \n",
       "1  [Interactive Marketing, Content Strategy, Affi...   \n",
       "2  [Primavera, Revit MEP, AutoCAD, Engineering, H...   \n",
       "3  [Talent Acquisition, Recruiting, Talent Manage...   \n",
       "4  [Brand Management, Integrated Marketing, Telev...   \n",
       "\n",
       "                                         specilities  \\\n",
       "0                                                      \n",
       "1  Internet Marketing, Interactive Marketing, Dig...   \n",
       "2                                                      \n",
       "3                                                      \n",
       "4                                                      \n",
       "\n",
       "                                           interests  \\\n",
       "0                  personal genomics, nanotechnology   \n",
       "1  Digital Photography, Tennis, Internet Marketin...   \n",
       "2  Project Management, Innovative Designs, New Te...   \n",
       "3  Organizational development, personal developme...   \n",
       "4   Broadcast Design, Television, Creativity, Music.   \n",
       "\n",
       "                                         institution      degree  \\\n",
       "0              [Harvard University, Yale University]   [PHD, BS]   \n",
       "1                           [University of Virginia]        [BA]   \n",
       "2  [University of Petroleum & Energy Studies, IIT...  [, , , , ]   \n",
       "3  [Universitatea „Transilvania” din Brașov, Univ...      [BA, ]   \n",
       "4                   [Instituto de Diseño de Caracas]          []   \n",
       "\n",
       "                                   major  \\\n",
       "0         [Biophysics, Computer Science]   \n",
       "1                              [History]   \n",
       "2                             [, , , , ]   \n",
       "3  [Psychology & Science of Education, ]   \n",
       "4                                     []   \n",
       "\n",
       "                                             summary  \\\n",
       "0  I am interested in inventing new methods to co...   \n",
       "1  Love to work on deadline & data-driven, dynami...   \n",
       "2  More than 08 yrs of experience with hands on e...   \n",
       "3  RecruitmentInternal communicationTrainingNew h...   \n",
       "4  Accomplished Chief Creative Officer with 15+ y...   \n",
       "\n",
       "                                              honors         industry  \\\n",
       "0                                                 []         Research   \n",
       "1                                                 []         Internet   \n",
       "2  [# Selected in ARMY through SSB (Technical Gra...     Oil & Energy   \n",
       "3                                                 []  Human Resources   \n",
       "4  [Promax and BDA World Gold Awards, New York Fe...  Broadcast Media   \n",
       "\n",
       "                                          occupation  \\\n",
       "0  [Assistant Professor, Technology Development F...   \n",
       "1  [Social Media Marketing Manager, Board of Dire...   \n",
       "2  [Manager (International Business Development &...   \n",
       "3  [Recruitment consultant, Scientific Staffing C...   \n",
       "4  [Creative Services, Vice President, Creative S...   \n",
       "\n",
       "                                             company year_of_work_experience  \\\n",
       "0  [UCSF, Wyss Institute for Biologically Inspire...                      16   \n",
       "1  [Coca-Cola, Atlanta Interactive Marketing Asso...                      21   \n",
       "2  [VOITH Hydro Pvt Ltd., VOITH Hydro Pvt. Ltd., ...                      22   \n",
       "3  [CGI, Kelly Services, Carmeuse, Education Inst...                      18   \n",
       "4  [Discovery Communications, Warner Channel, War...                      28   \n",
       "\n",
       "  group                                  latest occupation  \n",
       "0    []                                Assistant Professor  \n",
       "1    []                     Social Media Marketing Manager  \n",
       "2    []  Manager (International Business Development & ...  \n",
       "3    []                             Recruitment consultant  \n",
       "4    []                                  Creative Services  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a label column in the dataframe called \"latest occupation\"\n",
    "df['latest occupation'] = ''\n",
    "for index, row in df.iterrows():\n",
    "    row['latest occupation'] = row['occupation'][0]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assign a unique multiclass label to each person's latest occupation (0-49)\n",
    "latest_occupation = list()\n",
    "for i in df['latest occupation']:\n",
    "    r = list()\n",
    "    i=i.translate(str.maketrans('', '', string.punctuation))\n",
    "    i=i.replace(\" \", \"\")\n",
    "    i=i.lower()\n",
    "    r.append(i)\n",
    "    latest_occupation.append(r)\n",
    "\n",
    "all_sentencesY = []\n",
    "for text in latest_occupation:\n",
    "    all_sentencesY.append(list(text))\n",
    "    \n",
    "phrases = Phrases(all_sentencesY, min_count=1,threshold=1)\n",
    "bigram=Phraser(phrases)\n",
    "all_sentencesY = list(bigram[all_sentencesY])\n",
    "\n",
    "min_count = 5\n",
    "modelY = Word2Vec(all_sentencesY, \n",
    "                 min_count=min_count,   # Ignore words that appear less than this\n",
    "                 size=200,      # Dimensionality of word embeddings\n",
    "                 workers=16,     # Number of processors (parallelisation)\n",
    "                 window=5,      # Context window for words during training\n",
    "                 iter=30)       # Number of epochs training over corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "['frontofficemanager', 'director', 'senioroperationsmanager', 'assistantvicepresident', 'researchfellow', 'digitalmarketingmanager', 'creativedirector', 'projectarchitect', 'freelanceconsultant', 'quantitativeanalyst', 'directorofsales', 'technicalsalesmanager', 'principalassociate', 'assistantpropertymanager', 'seniortaxassociate', 'pastpresident', 'softwareengineerii', 'technicalarchitect', 'directorofengineering', 'vpofengineering', 'onlinemarketingmanager', 'contributingwriter', 'seniorandroiddeveloper', 'geologist', 'traineesolicitor', 'srhrmanager', 'graphicartist', 'workexperience', 'stylist', 'divisionmanager', 'modulelead', 'graduateresearchassistant', 'pricinganalyst', 'accountrepresentative', 'rdmanager', 'seniorhradvisor', 'territoryaccountmanager', 'humancapitalconsultant', 'headofdevelopment', 'customermarketingmanager', 'seniortaxmanager', 'financialaccountant', 'chiefgeologist', 'compensationmanager', 'creative', 'associatesoftwareengineer', 'strategymanager', 'acousticconsultant', 'researchscholar', 'designassistant', 'technician']\n",
      "\n",
      "\n",
      "['consultant', 'mechanicalengineer', 'financialanalyst', 'businessdevelopmentdirector', 'specialist', 'itconsultant', 'regionalmanager', 'seniorhumanresourcesgeneralist', 'salesmarketingmanager', 'taxmanager', 'eventcoordinator', 'regionalsalesmanager', 'managerhr', 'generalmanageroperations', 'businessdeveloper', 'securityanalyst', 'managingattorney', 'pharmacist', 'guestlecturer', 'creativeintern', 'headofbusinessdevelopment', 'productlinemanager', 'softwareconsultant', 'solutionarchitect', 'corporatesalesmanager', 'srvicepresident', 'assistanteditor', 'hrprofessional', 'eventsmanager', 'environmentalplanner', 'environmentalscientist', 'seniorauditmanager', 'juniorgraphicdesigner', 'practicemanager', 'internee', 'graduateengineer', 'fieldsalesmanager']\n",
      "\n",
      "\n",
      "['conceptartist', 'ceofounder', 'seniornetworkengineer', 'groupproductmanager', 'seoexecutive', 'vpsales', 'seniormemberoftechnicalstaff', 'javadeveloper', 'seniornetdeveloper', 'seniorresearcher', 'freelancegraphicdesigner', 'editorinchief', 'vptechnology', 'vpproduct', 'sde', 'adjunctlecturer', 'lawyer', 'srmanagerhr', 'senioroperationsengineer', 'taxassociate', 'keyaccountmanager', 'investmentbankingsummeranalyst', 'softwareengineeringmanager', 'salesadvisor', 'seniornegotiator', 'humanresourcegeneralist', 'chairmanoftheboard', 'actuarialconsultant', 'marketingassistant', 'accountstrategist', 'chiefcreditofficer', 'itanalyst', 'businessdevelopmentofficer', 'senioradviser', 'marketsalesmanager', 'groupmarketingmanager', 'researchcoordinator', 'seniorcounsel', 'directorofqualityassurance', 'charteredfinancialplanner', 'procurementengineer']\n",
      "\n",
      "\n",
      "['generalmanager', 'internetmarketingspecialist', 'branchmanager', 'itmanager', 'iosdeveloper', 'developer', 'databasemarketingmanager', 'principal', 'businessconsultant', 'marketingspecialist', 'marketingconsultant', 'lawclerk', 'humanresourcesdirector', 'projectplanner', 'promoter', 'ownerphotographer', 'lecturer', 'vicepresidentsales', 'chieffinancialofficer', 'webdesigner', 'researchintern', 'graduatestudent', 'ittechnician', 'researchexecutive', 'applicationsengineer', 'consultingmanager', 'internationalmarketingmanager', 'projectdirector', 'chiefstrategyofficer', 'mastersstudent', 'systemanalyst', 'vicepresidentofengineering', 'member', 'softwaredevelopmentmanager', 'seniorresearchanalyst', 'outsidesalesrepresentative', 'managingdirectorceo', 'sponsorshipmanager', 'geophysicist', 'seniorresearchengineer', 'actuarialanalyst', 'groupmanager', 'managerfinance', 'srprocessengineer', 'hardwareengineer', 'directorofquality', 'marketanalyst', 'testengineer', 'projectcontrolsmanager', 'srmanufacturingengineer', 'financialassistant', 'seniorcorporatecounsel', 'labassistant', 'reliabilityengineer', 'directorglobalmarketing', 'supplychainspecialist', 'psychotherapist', 'processmanager']\n",
      "\n",
      "\n",
      "['businesshead', 'headofoperations', 'associatevicepresident', 'seniorprojectmanager', 'productmanager', 'strategyconsultant', 'vpsalesmarketing', 'userexperiencedesigner', 'vicechairman', 'materialsmanager', 'softwareengineeringintern', 'humanresourcesgeneralist', 'adjunctfaculty', 'clientexecutive', 'marketresearchanalyst', 'mbacandidate2014', 'principalconsultant', 'freelancer', 'qualityassuranceengineer', 'mscstudent', 'mentor', 'mbastudent', 'seniorinteractivedesigner', 'interiordesigner', 'secretarygeneral', 'postdoctoralresearchfellow', 'vicepresidentproductmanagement', 'talentacquisitionconsultant', 'businessdevelopmentrepresentative', 'boardofdirectors', 'vicepresidentofsalesmarketing', 'chieftechnologyofficer', 'seniorprogramofficer', 'scientistii', 'postdoctoralresearchassistant', 'sharepointdeveloper', 'eventmanager', 'directorowner', 'srmechanicalengineer', 'structuraldesignengineer', 'registerednurse', 'hrexecutive', 'navalarchitect', 'uideveloper', 'vicepresidentofhumanresources', 'masterstudent', 'hotelmanager', 'barrister', 'webgraphicdesigner', 'managerinternalaudit', 'financeintern', 'regionalhrmanager', 'seniorhrbusinesspartner', 'companydirector', 'cateringsalesmanager', 'pa', 'srscientist', 'electricaldesignengineer', 'marketriskanalyst', 'regulatoryaffairsmanager']\n",
      "\n",
      "\n",
      "['pipelineengineer', 'softwaredevelopmentengineer', 'seniortestanalyst', 'seniorresearchassociate', 'directorpmo', 'controlsengineer', 'associatecreativedirector', 'technicalwriter', 'crewmember', 'characterartist', 'businessaccountmanager', 'webdesignerdeveloper', 'executiveproducer', 'pastor', 'accountsmanager', 'productionsupervisor', 'seniorportfoliomanager', 'consultingengineer', 'assistantdirectorofadmissions', 'qualitycontrolmanager', 'creativedirectorowner', 'legalassistant', 'seniorcommercialmanager', 'seniorquantitysurveyor', 'rfengineer', 'economist', 'environmentalspecialist', 'sustainabilitymanager', 'contactcentremanager', 'itoperationsmanager', 'model', 'assistantprincipal']\n",
      "\n",
      "\n",
      "['graphicdesigner', 'realestatebroker', 'seniorvicepresident', 'electricalengineer', 'engagementmanager', 'architecturaldesigner', 'staffingconsultant', 'directormarketing', 'chiefarchitect', 'teachingassistant', 'directorbusinessdevelopment', 'selfemployed', 'presidentfounder', 'technologylead', 'testanalyst', 'treasurymanager', 'corporaterecruiter', 'srfinancialanalyst', 'chairperson', 'founderpresidentceo', 'blogger', 'adviser', 'intransition', 'salesaccountmanager', 'affiliate', 'procurementspecialist', 'representative', 'directoroffoodandbeverage', 'executiveconsultant', 'dataengineer', 'businessunitmanager', 'chiefproductofficer', 'programmemanager', 'webeditor', 'solutionsconsultant', 'marketinganalyst', 'projectanalyst', 'directorofresearch', 'seniorvisualdesigner', 'srsystemsanalyst', 'developmentofficer', 'traininganddevelopmentmanager', 'legalconsultant', 'srgraphicdesigner', 'srdirectorofengineering', 'principalmechanicalengineer']\n",
      "\n",
      "\n",
      "['webmanager', 'researchanalyst', 'recruiter', 'computationalbiologist', 'humanresources', 'chiefengineer', 'digitalproducer', 'businessdevelopmentcoordinator', 'hrcoordinator', 'leaddesigner', 'srmanager', 'processengineer', 'humanresourcesbusinesspartner', 'freelance', '3dartist', 'staffaccountant', 'managingconsultant', 'seniormanagerfinance', 'graduateconsultant', 'srdesignengineer', 'seniordigitaldesigner', 'srdesigner', 'vpengineering', 'insidesalesrepresentative', 'informationtechnologymanager', 'seniormanagerdigitalmarketing', 'vicepresidentinformationtechnology', 'regionaloperationsmanager', 'contractadministrator', 'directorcorporatefinance', 'seniorsalesexecutive', 'flightattendant', 'fellow', 'plantmanager', 'digitaldirector', 'researchdirector', 'visitingprofessor', 'presidentcoo', 'srhrbusinesspartner', 'constructionprojectmanager', 'accountingassistant', 'itintern', 'compliancemanager', 'automationengineer', 'contractsadministrator', 'graduatemanagementtrainee']\n",
      "\n",
      "\n",
      "['estimator', 'environmentartist', 'principalsoftwareengineer', 'ownermanager', 'hranalyst', 'technologyanalyst', 'mbacandidateclassof2013', 'developmentcoordinator', 'industrialengineer', 'consultantdeveloper', 'productmarketingmanager', 'physicaldesignengineer', 'staffsoftwareengineer', 'socialworker', 'chiefriskofficer', 'seniorarchitect', 'md', 'generalsalesmanager', 'patentattorney', 'consultantsoftwareengineer', 'recruitingconsultant', 'recruitmentcoordinator', 'accountingmanager', 'writer', 'unemployed', 'insidesales', 'regionalvicepresident', 'vpfinance', 'businessintelligenceanalyst', 'srsoftwaredeveloper', 'foundingdirector', 'fieldserviceengineer', 'explorationgeologist', 'staffscientist', 'techleadmanager', 'projectspecialist', 'host', 'leveldesigner', 'trademarketingmanager', 'leadprogrammer', 'assistantbranchmanager', 'projectsmanager', 'fresher', 'environmentalconsultant', 'leadmechanicalengineer']\n",
      "\n",
      "\n",
      "['seniorsoftwareengineer', 'softwareengineer', 'directorofhumanresources', 'mbacandidate', 'ecommercemanager', 'residentassistant', 'adjunctinstructor', 'gameartist', 'boarddirector', 'enterprisearchitect', 'associateconsultant', 'seomanager', 'officeassistant', 'customerservicemanager', 'cofoundermanagingpartner', 'tutor', 'phd', 'hrrecruiter', 'relationshipmanager', 'actuarialassociate', 'technicalsalesrepresentative', 'seniorbuyer', 'salesconsultant', 'directorofmarketingcommunications', 'chairmanceo', 'generaldirector', 'committeemember', 'careercounselor', 'principalarchitect', 'recruitingmanager', 'businessdevelopmentintern', 'seniordevelopmentmanager', 'retired', 'technicalproductmanager', 'completionsengineer', 'directoroffinance', 'projectofficer', 'leadelectricalengineer', 'managerfinancialplanninganalysis', 'sraccountmanager', 'managingmember', 'seniormanagermarketing', 'directorinternalaudit', 'manageroperations', 'salesoperationsmanager', 'applicationsmanager', 'litigationassociate', 'chefdecuisine', 'siteengineer', 'chemist', 'independentfinancialadviser']\n",
      "\n",
      "\n",
      "['marketingmanager', 'staffengineer', 'presidentowner', 'qualityengineer', 'seniorartdirector', 'ofcounsel', 'founderprincipal', 'investmentbankinganalyst', 'leadarchitect', 'productdevelopmentmanager', 'seniorfrontenddeveloper', 'coordinator', 'assistantfrontofficemanager', 'seniorrecruiter', 'teamleader', 'postdoctoralassociate', 'marketingcommunicationsmanager', 'seniorreservoirengineer', 'productdevelopment', 'seniorelectricalengineer', 'captain', 'vpoperations', 'vicepresidentgeneralmanager', 'landscapedesigner', 'counsellor', 'departmentmanager', 'facilitiesengineer', 'qualityassurance', 'svp', 'councilmember', 'communicationscoordinator', 'vicepresidentpartner', 'caddesigner', 'memberboardofdirectors', 'projectadministrator', 'humanresource', 'realestateinvestor', 'businessunitdirector', 'seniorproducer', 'commander', 'founderandmanagingpartner', 'accountingclerk', 'chiefinformationofficercio', 'channelmanager', 'assistantaccountant', 'electrician']\n",
      "\n",
      "\n",
      "['assistantteacher', 'assistantmanager', 'projectmanager', 'seniorengineer', 'associateattorney', 'businessdevelopment', 'seniorauditor', 'presidentcofounder', 'operationsmanager', 'accountsassistant', 'systemsmanager', 'financialadvisor', 'technicalsupportengineer', 'fieldengineer', 'nationalaccountmanager', 'assistantsalesmanager', 'majoraccountmanager', 'technicalrecruiter', 'communicationsspecialist', 'sranalyst', 'internalcommunicationsmanager', 'facilitator', 'ownerceo', 'vp', 'businessdirector', 'adjunctprofessor', 'environmentalengineer', 'developmentlead', 'directorhumanresources', 'technicalconsultant', 'assistantaccountexecutive', 'estimatorprojectmanager', 'engineeringintern', 'regionalprojectmanager', 'foundermanagingpartner', 'seniortechnicalrecruiter', 'postdoctoralscholar', 'technicalmanager', 'continuousimprovementmanager', 'graduateteachingassistant', 'parttimefaculty', 'seniordataanalyst', 'directorofcareerservices', 'staffwriter', 'clientpartner', 'directorofnationalaccounts', 'presalesconsultant', 'operationsengineer', 'seniorlegalcounsel']\n",
      "\n",
      "\n",
      "['managementtrainee', 'engineeringmanager', 'accountexecutive', 'productengineer', 'recruitmentmanager', 'supplychainmanager', 'analyst', 'technicalprojectmanager', 'brokerowner', 'salesmanager', 'seniorprogrammanager', 'teammanager', 'vicepresidentofsales', 'loanofficer', 'vicepresidentengineering', 'research', 'programassistant', 'vicepresidentofbusinessdevelopment', 'ceopresident', 'realestateagent', 'businessspecialist', 'seniorauditassociate', 'senioractuarialanalyst', 'informationsecurityspecialist', 'hrassociate', 'directortalentacquisition', 'mbacandidate2013', 'seniorstrategist', 'investmentadvisor', 'graphicdesignintern', 'vicepresidentofdevelopment', 'financialanalystintern', 'associateprogrammanager', 'trainingconsultant', 'hrpartner', 'investmentconsultant', 'financialservicesrepresentative']\n",
      "\n",
      "\n",
      "['researcher', 'executivedirector', 'productmarketing', 'uidesigner', 'accountant', 'architect', 'assistantcontroller', 'coowner', 'seniorrelationshipmanager', 'humanresourcescoordinator', 'projectexecutive', 'chiefinformationsecurityofficer', 'author', 'broker', 'globalaccountmanager', 'seniormembertechnicalstaff', 'talentmanager', 'directorqualityassurance', 'shareholder', 'seniormanagementconsultant', 'seniorexecutive', 'seniorgeologist', 'chiefofstaff', 'trainingcoordinator', 'deputydirector', 'seniorgraphicdesigner', 'animator', 'insidesalesmanager', 'deputygeneralmanager', 'auditintern', 'exportmanager', 'hydrogeologist', 'seniorstatisticalprogrammer', 'casemanager', 'maintenancemanager', 'seniorprojectplanner', 'executivehr', 'freelanceartdirector', 'projectdesigner', 'headhr', 'humanresourceofficer', 'directorofrooms', 'safetymanager', 'server']\n",
      "\n",
      "\n",
      "['assistantprofessor', 'managersales', 'chiefoperatingofficer', 'vicepresidentofoperations', 'seniortestengineer', 'hrrecruitmentspecialist', 'talentacquisitionmanager', 'fundmanager', 'eventplanner', 'teamlead', 'postdoc', 'creativetechnologist', 'directorofdevelopment', 'administrator', 'independentdistributor', 'hrgeneralist', 'seniorbusinessanalyst', 'conferencedirector', 'qualitymanager', 'directorproductmanagement', 'technicalaccountmanager', 'globalmarketingmanager', 'designdirector', 'technicalofficer', 'ownerpartner', 'seniorresearchscientist', 'srwebdeveloper', 'licensedrealestateagent', 'seniorsystemsadministrator', 'researchconsultant', 'directoroperations', 'juniorconsultant', 'webmaster', 'agencyowner', 'seniorhrmanager', 'rubydeveloper', 'internarchitect', 'executive', 'assistant', 'seniorresearchmanager', 'superintendent', 'businesssystemsanalyst', 'gamedeveloper', 'seniorresearchfellow', 'businessintelligencemanager', 'planningmanager', 'softwaretester', 'seniorgeotechnicalengineer', 'technicalstaff', 'administrativesupport', 'senior', 'privateclientadviser', 'publicrelationsmanager', 'communicationsofficer', 'officecoordinator', 'projectquantitysurveyor', 'structuraldesigner', 'srhumanresourcesgeneralist', 'wealthadvisor', 'assistantmanagerfinance']\n",
      "\n",
      "\n",
      "['executiveadministrativeassistant', 'cofounderandceo', 'mbacandidateclassof2014', 'landscapearchitect', 'hrspecialist', 'systemsarchitect', 'vicepresidentbusinessdevelopment', 'storemanager', 'softwaredeveloperengineer', 'founderpresident', 'seniorstructuralengineer', 'fulltimestudent', 'seniormedicalwriter', 'mobiledeveloper', 'groupcfo', 'administrativeofficer', 'seniormanufacturingengineer', 'technologymanager', 'doctoralstudent', 'asstprofessor', 'managingassociate', 'associatepartner', 'directorofproduct', 'independentconsultant', 'optometrist', 'graphicwebdesigner', 'regionalhumanresourcesmanager', 'geotechnicalengineer', 'gm', 'areavicepresident', 'clientdevelopmentdirector', 'marketdevelopmentmanager', 'retailmanager', 'registeredrepresentative', 'administrationassistant', 'pipingengineer']\n",
      "\n",
      "\n",
      "['researchscientist', 'doctoralresearcher', 'legalcounsel', 'vpofoperations', 'seniorjavadeveloper', 'seniorrecruitmentconsultant', 'commercialdirector', 'sales', 'uxdesigner', 'proposalmanager', 'socialmediaspecialist', 'trainingspecialist', 'seniormarketingmanager', 'graduatestudentresearcher', 'strategicaccountmanager', 'informationtechnologydirector', 'districtsalesmanager', 'directorsalesmarketing', 'areabusinessmanager', 'paralegal', 'gamedesigner', 'financialplanner', 'datascientist', 'operationsanalyst', 'clinicalpsychologist', 'procurementmanager', 'revenueanalyst', 'foundercreativedirector', 'hrofficer', 'srengineeringmanager', 'seniorprojectengineer', 'dataarchitect', 'itdirector', 'directorofadmissions', 'softwaredesigner', 'sourcingspecialist', 'associateresearchscientist', 'componentdesignengineer', 'studentintern', 'auditseniorassociate', 'payrolladministrator', 'distributor', 'changemanager', 'documentcontroller', 'officer']\n",
      "\n",
      "\n",
      "['areasalesmanager', 'associategeneralcounsel', 'boardmember', 'postdoctoralfellow', 'salesandmarketing', 'businessdevelopmentmanager', 'headbusinessdevelopment', 'interactivedesigner', 'nationalsalesdirector', 'digitaldesigner', 'recruitmentadvisor', 'humanresourcesanalyst', 'seniorsoftwaredevelopmentmanager', 'visualdesigner', 'avp', 'communicationsdirector', 'managerrecruitment', 'assistantdirector', 'traineeaccountant', 'frontendengineer', 'salesprofessional', 'chiefoperationsofficer', 'technicalspecialist', 'headofproductdevelopment', 'accountinganalyst', 'productdeveloper', 'associateproductmanager', 'directorandfounder', 'barmanager', 'juniorarchitect', 'corporatecommunicationsmanager', 'bioinformaticsscientist', 'businessperformanceadvisor', 'franchiseowner', 'directorcommunications', 'clientrelationshipmanager', 'financeofficer', 'professional', 'srdeveloper', 'customerrelationshipmanager', 'regionalmanagerwest', 'subjectmatterexpert', 'administrationmanager', 'projectcontrolsspecialist']\n",
      "\n",
      "\n",
      "['softwaredeveloper', 'leadconsultant', 'informationarchitect', 'foundermanagingdirector', 'chiefinvestmentofficer', 'managerbusinessdevelopment', 'regionaldirector', 'userexperienceconsultant', 'seniorbusinessdevelopmentmanager', 'clientservicemanager', 'claimsmanager', 'executivesearchconsultant', 'accountspecialist', 'owneroperator', 'humanresourcesadvisor', 'taxstaff', 'testmanager', 'managerfinancialservices', 'taxaccountant', 'cashier', 'productioncoordinator', 'wealthmanager', 'executiveofficer', 'seniorplanner', 'digitalaccountmanager', 'vicepresidentsalesandmarketing', 'drillingengineer']\n",
      "\n",
      "\n",
      "['recruitmentconsultant', 'founderceo', 'seniorsoftwaredeveloper', 'vpofmarketing', 'founderandceo', 'cofoundercto', 'salessupport', 'personalassistant', 'directorofproductmarketing', 'attorney', 'salesspecialist', 'marketingexecutive', 'hradministrator', 'seniorhrconsultant', 'programofficer', 'territorymanager', 'teammember', 'leadsoftwareengineer', 'principalsoftwarearchitect', 'associateeditor', 'chiefscientist', 'webanalyst', 'webproducer', 'executiveeditor', 'businessrelationshipmanager', 'substituteteacher', 'experiencedassociate', 'deputymanagerhr', 'assistantmanagerhumanresources', 'regionalbusinessdevelopmentmanager', 'managermarketingcommunications', 'founderexecutivedirector', 'researchassistantprofessor', 'financecontroller', 'nursemanager', 'miningengineer', 'graduatetrainee', 'englishteacher', 'physician']\n",
      "\n",
      "\n",
      "['humanresourcesconsultant', 'businessanalyst', 'designer', 'financemanager', 'businessdevelopmentexecutive', 'webapplicationdeveloper', 'directorsales', 'directorengineering', 'vicepresidentsalesmarketing', 'propertyaccountant', 'researchstudent', 'seniorartist', 'applicationsdeveloper', 'regionalmarketingmanager', 'directorofsalesmarketing', 'socialmediacoordinator', 'embeddedsoftwareengineer', 'seniorassociateaudit', 'supportanalyst', 'advisor', 'careerbreak', 'businessprocessanalyst', 'productionengineer', 'relationshipofficer', 'seniorphpdeveloper', 'directorsalesoperations', 'visitingscholar', 'srengineer', 'seniorqualityengineer', 'uiuxdesigner', 'srmarketingmanager', 'structuralengineer', 'assistantgeneralcounsel', 'foundingpartner', 'recruitmentspecialist', 'evp', 'securitymanager', 'urbanplanner', 'taxconsultant', 'assistantbuyer', 'sourcingmanager', 'auditassociate', 'developmentteamlead', 'assistantgeneralmanager', 'customersupport', 'physicaltherapist', 'recruitingdirector', 'salessupervisor', 'clientadvisor', 'cadtechnician', 'srprojectengineer']\n",
      "\n",
      "\n",
      "['directorofmarketing', 'seniordeveloper', 'softwarearchitect', 'seniordesigner', 'auditor', 'projectleader', 'humanresourcemanager', 'solicitor', 'frontenddeveloper', 'plantcontroller', 'independentbusinessowner', 'hradvisor', 'investorrelations', 'salescoordinator', 'associatebroker', 'vpofsales', 'ownerfounder', 'salesdevelopmentmanager', 'srproductmarketingmanager', 'equityresearchassociate', 'seniorinvestmentanalyst', 'businessdevelopmentspecialist', 'senioranimator', 'seniorhumanresourcesmanager', 'surveyor', 'groupfinancialcontroller', 'humanresourcesofficer', 'srexecutive', 'seniormanagerhr', 'seniorminegeologist', 'laboratorymanager', 'leadsoftwaredeveloper', 'directoroffoodbeverage', 'seniorproductdevelopmentengineer', 'manageraccounts']\n",
      "\n",
      "\n",
      "['presidentceo', 'seniorproductmanager', 'videoeditor', 'strategist', 'executiveassistant', 'projectengineeringmanager', 'projectengineer', 'directorclientservices', 'vpbusinessdevelopment', 'techlead', 'engineer', 'principalscientist', 'gameprogrammer', 'marketingofficer', 'seniorcorporaterecruiter', 'seniorestimator', 'directorofsoftwaredevelopment', 'seniorscientist', 'regionalaccountmanager', 'developmentteamleader', 'teacher', 'entrepreneurinresidence', 'clientservicesmanager', 'managingprincipal', 'systemsanalyst', 'seniorloanofficer', 'seniormanagingconsultant', 'designerdeveloper', 'summerintern', 'applicationdeveloper', 'programdirector', 'contractrecruiter', 'infrastructuremanager', 'rn', 'qualityassurancemanager', 'areamanager', 'seniororacledba', 'clientaccountmanager', 'financeanalyst', 'manufacturingmanager', 'hrprojectmanager', 'eventscoordinator', 'copywriter', 'srhumanresourcesmanager', 'associatepastor', 'chiefmedicalofficer', 'marketingandcommunicationsmanager', 'sharepointconsultant', 'researchprofessor', 'gisanalyst', 'visualmerchandiser', 'itsupport']\n",
      "\n",
      "\n",
      "['netdeveloper', 'phdcandidate', 'cio', 'humanresourcesmanager', 'salesmarketingdirector', 'chairman', 'directorofbusinessdevelopment', 'generalcounsel', 'principalowner', 'secretary', 'srmanagerbusinessdevelopment', 'fashiondesigner', 'restaurantmanager', 'technicaldirector', 'ambassador', 'illustrator', 'seniorprocessengineer', 'mba', 'salesperson', 'principalresearchscientist', 'humancapitalanalyst', 'seniorgeophysicist', 'commercialfinancemanager', 'grader', 'learningdevelopmentmanager', 'dutymanager']\n",
      "\n",
      "\n",
      "['medicalwriter', 'systemsengineer', 'cto', 'seniorqualityassuranceengineer', 'businessdevelopmentassociate', 'memberoftechnicalstaff', 'visitingfaculty', 'ceocofounder', 'deputymanager', 'territorysalesmanager', 'petroleumengineer', 'solutionsengineer', 'internalauditor', 'associateintern', 'legaladvisor', 'manufacturingengineeringmanager', 'executiverecruiter', 'constructionmanager', 'juniordesigner', 'pmomanager', 'digitalmarketingexecutive', 'automationspecialist', 'businessowner', 'headofresearch', 'researchanalystintern', 'regionalgeneralmanager', 'hairstylist', 'healthandsafetymanager']\n",
      "\n",
      "\n",
      "['srassociate', 'industrialdesigner', 'technicalsales', 'certifiedfinancialplanner', 'payrollmanager', 'networkengineer', 'seniormanagerbusinessdevelopment', 'regionalsalesexecutive', 'freelanceillustrator', 'assistantprojectmanager', 'vpsalesandmarketing', 'freelancedesigner', 'seniorsalesconsultant', 'educator', 'seniormanagerhumanresources', 'chiefmarketingofficer', 'entrepreneur', 'electricalengineeringmanager', 'asstvicepresident', 'presidentandceo', 'directorsoftwareengineering', 'taxpartner', 'mortgageloanofficer', 'headoffinance', 'vicepresidentoperations', 'headofit', 'litigationparalegal', 'sitemanager', 'medicaldirector', 'enterpriserelationshipmanager', 'digitalmanager', 'analystdeveloper', 'professorofcomputerscience', 'visitingfellow', 'associateanalyst', 'training', 'associatebrandmanager', 'maanalyst', 'hseadvisor']\n",
      "\n",
      "\n",
      "['technicalleader', 'vicepresident', 'trustee', 'newbusinessdevelopmentmanager', 'investmentmanager', 'advisoryboardmember', 'seniorsystemsengineer', 'financialrepresentative', 'senioraccountmanager', 'customerservicerepresentative', 'cdeveloper', 'benefitsmanager', 'managersoftwareengineering', 'managerhumanresources', 'directormarketingcommunications', 'researchmanager', 'iossoftwareengineer', 'senioruserexperiencedesigner', 'managertalentacquisition', 'pharmacymanager', 'senioraccountexecutive', 'vicepresidentoffinance', 'systemengineer', 'seniorfellow', 'analystintern', 'networkadministrator', 'directorsalesandmarketing', 'founderowner', 'taxsupervisor', 'faculty', 'outsidesales', 'srtechnicalrecruiter', 'bookkeeper', 'visitingassistantprofessor', 'soleproprietor', 'seniorsalesrepresentative', 'chiefdesigner', 'electricaldesigner', 'principaldesigner', 'technicalsupportmanager', 'accountingintern', 'seniortaxaccountant', 'seniorresearchexecutive', 'srelectricalengineer', 'marketingcommunicationscoordinator']\n",
      "\n",
      "\n",
      "['president', 'srconsultant', 'seniorapplicationdeveloper', 'photographer', 'resourcemanager', 'productspecialist', 'cofounderanddirector', 'staffingmanager', 'marketingdirector', 'salesassistant', 'volunteer', 'cofounderdirector', 'reservoirengineer', 'petrophysicist', 'brandmanager', 'graduateassistant', 'doctoralcandidate', 'districtmanager', 'hrassistant', 'financialconsultant', 'freelanceproducer', 'reporter', 'cochair', 'seniorwealthmanager', 'testlead', 'facilitymanager', 'taxsenior', 'businessarchitect', 'executivemanager', 'creativedesigner', 'engineeringprojectmanager', 'memberoftheboardofdirectors', 'personalbanker', 'onlinemarketingexecutive', 'assistantportfoliomanager', 'hsemanager', 'mr']\n",
      "\n",
      "\n",
      "['artdirector', 'associateprofessor', 'seniorlecturer', 'contractsmanager', 'ceoandcofounder', 'operationsdirector', 'presidentandfounder', 'interactivedeveloper', 'productionassistant', 'associateprojectmanager', 'businesstechnologyanalyst', 'nationaloperationsmanager', 'headchef', 'assistantbrandmanager', 'directorfinance', 'directorofevents', 'ctofounder', 'computerscientist', 'treasuryanalyst', 'environmentalmanager', 'seniorresearchconsultant', 'hrsupervisor', 'trainer', 'membershipmanager', 'actuary']\n",
      "\n",
      "\n",
      "['summerassociate', 'softwareengineerintern', 'keyaccountsmanager', 'seniorassociate', 'solutionsarchitect', 'humanresourcesrepresentative', 'programcoordinator', 'seniorproductmarketingmanager', 'frontendwebdeveloper', 'agent', 'underwriter', 'generalpartner', 'foundercto', 'assurancesenior', 'seniorpartner', 'seniorfinancemanager', 'hr', 'systemsengineeringmanager', 'careercoach', 'contractattorney', 'internetmarketingmanager', 'analystprogrammer', 'associatesolicitor', 'sixsigmablackbelt', 'regulatoryaffairsspecialist', 'architecturalassistant', 'seniorvicepresidentsales', 'projectaccountant', 'seniorcontractsengineer', 'divisionaldirector']\n",
      "\n",
      "\n",
      "['corporateaccountmanager', 'ownercreativedirector', 'managingpartner', 'student', 'vicepresidentfinance', 'founder', 'softwaretestengineer', 'researchofficer', 'investigator', 'directortalentmanagement', 'designmanager', 'securityconsultant', 'legalintern', 'journalist', 'talentacquisition', 'seniormanagersoftwaredevelopment', 'productmanagement', 'managingeditor', 'analyticalchemist', 'productdirector', 'seniorinternalauditor', 'associatedesigner', 'managersoftwaredevelopment', 'chiefoperatingofficercoo', 'projectgeologist', 'corporatecontroller', 'membertechnicalstaff', 'juniorassociate', 'qualityassurancespecialist', 'legalsecretary', 'aerospaceengineer', 'chieffinancialofficercfo', 'financialadviser', 'counselor', 'customerservice', 'projectmanagerestimator', 'srdirector', 'salesofficer']\n",
      "\n",
      "\n",
      "['associate', 'ceo', 'contractor', 'seniorfinancialanalyst', 'ownerdirector', 'commercialmanager', 'servicedeliverymanager', 'treasurer', 'majorgiftsofficer', 'managementaccountant', 'characteranimator', 'teller', 'scientist', 'organizer', 'customersupportmanager', 'productionartist', 'principalprocessengineer', 'vicepresidentcorporatedevelopment', 'visitinglecturer', 'grouphrmanager', 'securityengineer', 'taxseniormanager', 'seniortaxconsultant', 'graduatemechanicalengineer', 'managingdirectorowner', 'healthsafetymanager']\n",
      "\n",
      "\n",
      "['categorymanager', 'officemanager', 'realestateconsultant', 'freelancewriter', 'associateproducer', 'administrativeassistant', 'cpa', 'vpofbusinessdevelopment', 'leadpastor', 'chiefaccountant', 'headofmarketing', 'civilengineer', 'juniorengineer', 'drillingsupervisor', 'instructor', 'financialservicesprofessional', 'assetmanager', 'teachingfellow', 'assistantstoremanager', 'sraccountexecutive', 'managerhumanresource', 'administrativecoordinator', 'portfolioanalyst', 'companysecretary', 'technicalassistant']\n",
      "\n",
      "\n",
      "['owner', 'seniorsoftwaredevelopmentengineer', 'researchassociate', 'associateengineer', 'controller', 'programmeranalyst', 'srprojectmanager', 'phpdeveloper', 'pressofficer', 'marketingintern', 'validationengineer', 'venturepartner', 'regionaldirectorofsales', 'founderandprincipal', 'seniordevelopmentengineer', 'assuranceassociate', 'souschef', 'srhrgeneralist', 'clinicalresearchassociate', 'investor', 'directorsupplychain', 'maintenancesupervisor']\n",
      "\n",
      "\n",
      "['seniorsoftwarearchitect', 'seniorproductdeveloper', 'directorofproductdevelopment', 'salesexecutive', 'planner', 'digitalprojectmanager', 'financedirector', 'investmentprofessional', 'investmentanalyst', 'directorofsalesandmarketing', 'freelancewebdesigner', 'nationalaccountexecutive', 'seniorsolutionsarchitect', 'seniorsystemanalyst', 'associatebuyer', 'coach', 'corporatedevelopment', 'cofoundercoo', 'brokerassociate', 'marketing', 'supportengineer', 'audioengineer', 'itspecialist', 'presidentgeneralmanager', 'supplychainanalyst', 'assistantengineer', 'safetycoordinator']\n",
      "\n",
      "\n",
      "['managingdirector', 'partner', 'producer', 'associatedirector', 'seniorconsultant', 'founderanddirector', 'managermarketing', 'manufacturingengineer', 'cofounderceo', 'centremanager', 'contractmanager', 'headmarketing', 'portfoliomanager', 'seniordirector', 'salesandmarketingmanager', 'hrconsultant', 'vpproductmarketing', 'trader', 'publisher', 'businessanalystprojectmanager', 'srhrexecutive', 'managingpartnerfounder', 'facilitiesmanager', 'safetyadvisor', 'businesssolutionsmanager', 'projectconsultant', 'projectcontrolsengineer', 'actress', 'executivecoach', 'managerregulatoryaffairs', 'directorofcustomerservice', 'riskmanager', 'headofhr', 'associatemarketingmanager', 'humanresourcesadministrator', 'productanalyst', 'libraryassistant']\n",
      "\n",
      "\n",
      "['researchassistant', 'senioraccountant', 'salesengineer', 'coo', 'chiefinformationofficer', 'executiveinresidence', 'salesassociate', 'artisticdirector', 'softwareengineeriii', 'complianceofficer', 'marketingcommunications', 'networksecurityengineer', 'actor', 'executivechairman', 'accountcoordinator', 'directorit', 'managingdirectorfounder', 'ownerprincipal', 'pilot', 'mechanicaldesigner', 'businessoperationsmanager', 'seniordirectorbusinessdevelopment', 'corporateaccountant', 'careeradvisor', 'communicationsadvisor', 'chiefdevelopmentofficer', 'senioreditor', 'ownerinteriordesigner', 'humanresourcesintern', 'presenter', 'commercialunderwriter']\n",
      "\n",
      "\n",
      "['seniordesignengineer', 'hrmanager', 'nonexecutivedirector', 'phdstudent', 'vicepresidentmarketing', 'researchstaffmember', 'receptionist', 'realtor', 'androiddeveloper', 'technologist', 'directorofit', 'hrdirector', 'technicallead', 'senioraccountdirector', 'ecommercemarketingmanager', 'intelligenceanalyst', 'senioritconsultant', 'regionalbusinessmanager', 'librarian', 'waitress', 'undergraduateresearcher', 'tourguide', 'countrysalesmanager', 'lettingsmanager']\n",
      "\n",
      "\n",
      "['postdoctoralresearcher', 'cofounder', 'seniorinteractiondesigner', 'editor', 'chair', 'staff', 'seniormechanicalengineer', 'humanresourcebusinesspartner', 'directorhr', 'postgraduatestudent', 'equityresearchanalyst', 'leadproductmanager', 'productionmanager', 'financialcontroller', 'talentacquisitionspecialist', 'seniormanagingdirector', 'associateprincipal', 'projectsuperintendent', 'resourcingconsultant', 'internationalsalesmanager', 'seniorindustrialdesigner', 'equityanalyst', 'directorproductmarketing', 'seniorbrandmanager', 'studentambassador', 'communicationsconsultant', 'quantitysurveyor']\n",
      "\n",
      "\n",
      "['manager', 'internalauditmanager', 'chiefexecutiveofficer', 'developmentdirector', 'seniordatabaseadministrator', 'investmentbankingassociate', 'ownerconsultant', 'ceoowner', 'headofdesign', 'officeadministrator', 'vphumanresources', 'creativeconsultant', 'juniorsoftwaredeveloper', 'seoconsultant', 'clinicalprojectmanager', 'creditriskanalyst', 'headofsales', 'accountingspecialist', 'riskanalyst', 'auditassistant', 'shiftsupervisor', 'costaccountant', 'countrydirector', 'seniorrecruitmentadvisor', 'hradminmanager', 'seniorpolicyadvisor', 'minegeologist', 'chefdepartie']\n",
      "\n",
      "\n",
      "['trainingmanager', 'ownerandfounder', 'seniorwebdeveloper', 'headofsalesmarketing', 'columnist', 'vicepresidenthumanresources', 'itrecruiter', 'independentcontractor', 'directorofstrategy', 'ownerdesigner', 'marketingassociate', 'codirector', 'vpproductdevelopment', 'headofdigital', 'seniormanagerproductmarketing', 'attorneyatlaw', 'developmentengineer', 'informationsystemsmanager', 'maintenancesuperintendent', 'manufacturingsupervisor', 'buildingsurveyor', 'learninganddevelopmentmanager', 'headhumanresources', 'revenuemanager', 'assistantmarketingmanager']\n",
      "\n",
      "\n",
      "['salesrepresentative', 'managementconsultant', 'webdeveloper', 'salesdirector', 'srproductmanager', 'servicemanager', 'rubyonrailsdeveloper', 'directorofoperations', 'applicationengineer', 'directorofcommunications', 'propertymanager', 'instructionaldesigner', 'associatecounsel', 'undergraduateresearchassistant', 'executivevicepresident', 'merchandiseplanner', 'proprietor', 'taxintern', 'administrativemanager', 'marketmanager', 'marketingcommunicationsdirector', 'mediaconsultant', 'dataanalyst', 'principle', 'seniorsalesmanager', 'vicepresidentofmarketing', 'bartender', 'seniorsalesspecialist', 'srloanofficer', 'ownermanagingdirector', 'generalmanagerhumanresources', 'projectsdirector', 'landman']\n",
      "\n",
      "\n",
      "['auditmanager', 'designengineer', 'principalengineer', 'interactiondesigner', 'directorofproductmanagement', 'founderandpresident', 'countrymanager', 'systemsadministrator', 'designconsultant', 'accountspayable', 'marketingcommunicationsspecialist', 'humanresourcesassistant', 'directorofproduction', 'medicalrepresentative', 'chiefexecutive', 'merchandisingmanager', 'socialmediamanager', 'databaseadministrator', 'areadirector', 'seniorsalesengineer', 'vicepresidentofproductmanagement', 'seniortechnicalconsultant', 'operationmanager', 'buyer', 'designintern', 'assistantquantitysurveyor', 'driver', 'itdevelopmentmanager', 'managerprocurement', 'salesrep', 'internalwholesaler']\n",
      "\n",
      "\n",
      "['leadengineer', 'commissioner', 'businessmanager', 'assistantmanagerhr', 'srprogrammanager', 'taxdirector', 'qaengineer', 'directorofclientservices', 'headofhumanresources', 'privatebanker', 'artist', 'postdocresearcher', 'deliverymanager', 'associatecorporatefinance', 'srrecruiter', 'executiveboardmember', 'mortgageloanoriginator', 'seniorriskanalyst', 'supplierqualityengineer', 'productdesigner', 'humanresourcesspecialist', 'seniorvicepresidenthumanresources', 'finance', 'consultingassociate', 'operations', 'seniorprogrammeranalyst', 'recruitingspecialist', 'employee', 'bidmanager', 'engineeringtechnician', 'dentist', 'seniorsurveyor', 'executivesouschef']\n",
      "\n",
      "\n",
      "['cofounderandcto', 'phdresearcher', 'accountmanager', 'foundingmember', 'financialmanager', 'projectassociate', 'internship', 'campaignmanager', 'assistantresearchprofessor', 'javasoftwareengineer', 'vicepresidentclientservices', 'purchasingmanager', 'directorsoftwaredevelopment', 'directoroftechnology', 'executivechef', 'managementassociate', 'auditseniormanager', 'directorinformationtechnology', 'qualityassuranceanalyst', 'controlsystemsengineer', 'presidentandchiefoperatingofficer', 'referencelibrarian', 'studiomanager', 'medicalstudent', 'seniorinteriordesigner']\n",
      "\n",
      "\n",
      "['itprojectmanager', 'seniormanager', 'commercialaccountmanager', 'developmentmanager', 'generalmanagerbusinessdevelopment', 'accountdirector', 'groupleader', 'compensationanalyst', 'enterpriseaccountexecutive', 'systemadministrator', 'asstmanager', 'qalead', 'marketingeventsmanager', 'graduateintern', 'technicalserviceengineer', 'graduateresearcher', 'associatemanager', 'salesandmarketingdirector', 'charteredaccountant', 'operationscoordinator', 'consultingactuary']\n",
      "\n",
      "\n",
      "['professor', 'intern', 'staffingspecialist', 'purchasingdirector', 'trainee', 'communicationsmanager', 'oracledba', 'investmentdirector', 'leadprocessengineer', 'rdengineer', 'executiveadvisor', 'technicalanalyst', 'architecturalintern', 'seniordrillingengineer', 'barristeratlaw', 'lookingforjob']\n",
      "\n",
      "\n",
      "['mechanicaldesignengineer', 'adjunctassistantprofessor', 'supervisor', 'projectcoordinator', 'volunteercoordinator', 'seniorstaffengineer', 'srsoftwareengineer', 'seniorprogrammer', 'researchengineer', 'creditanalyst', 'corporatecounsel', 'merchandiser', 'projectmanagement', 'salesmarketing', 'deputyeditor', 'vpmarketing', 'summeranalyst', 'mortgagebanker', 'operatingpartner', 'planningengineer', 'productiondirector', 'nationalaccountsmanager', 'directorprojectmanagement', 'studentassistant', 'recruitmentofficer', 'auditsupervisor']\n",
      "\n",
      "\n",
      "['clientservicesdirector', 'nationalsalesmanager', 'founderdirector', 'cfo', 'programmanager', 'seniorsystemengineer', 'technicalsupport', 'softwaredevelopmentengineerii', 'seniorwebdesigner', 'programmer', 'freelancedeveloper', 'regionalsalesdirector', 'insuranceagent', 'systemarchitect', 'ceoandfounder', 'srsystemsengineer', 'logisticsmanager', 'seniorapplicationsengineer', 'contributor', 'graduate', 'warehousemanager', 'fundaccountant', 'cra']\n",
      "\n",
      "\n",
      "['analystinvestmentbanking', 'postdoctoralresearchassociate', 'senioranalyst', 'marketingcoordinator', 'communitymanager', 'chieftechnicalofficer', 'counsel', 'businessdevelopmentconsultant', 'recruitingcoordinator', 'senioradvisor', 'hrbusinesspartner', 'medicalscienceliaison', 'auditsenior', 'leaddeveloper', 'headofproduct', 'boardofadvisors', 'projectlead', 'ownerpresident', 'headoftechnology', 'directorofinformationtechnology', 'qamanager', 'engineeringdirector', 'seniorassociateconsultant', 'productdevelopmentengineer', 'seniorunderwriter', 'clientmanager', 'recruitmentexecutive', 'srbusinessanalyst', 'seniorengineeringmanager', 'seniorsolicitor', 'seniorclinicalresearchassociate', 'projectassistant', 'procurementcoordinator', 'categorydevelopmentmanager']\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Hierarchical clustering\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "\n",
    "n_clusters = n_cluster_label\n",
    "cluster = AgglomerativeClustering(n_clusters=n_clusters, affinity='euclidean', linkage='ward') \n",
    "\n",
    "modelYvec_list = list()\n",
    "for i in list(modelY.wv.vocab.keys()):\n",
    "    modelYvec_list.append(modelY[i])\n",
    "modelYvec_cluster = list(cluster.fit_predict(modelYvec_list))\n",
    "\n",
    "modelYvec = dict()\n",
    "for i in range(len(modelY.wv.vocab.keys())):\n",
    "    modelYvec[list(modelY.wv.vocab.keys())[i]] = modelYvec_cluster[i]\n",
    "\n",
    "print('\\n')\n",
    "modelY_cluster_list = list()\n",
    "for i in range(0,n_clusters):\n",
    "    print([k for k,v in modelYvec.items() if v == i])\n",
    "    modelY_cluster_list.append([k for k,v in modelYvec.items() if v == i])\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of distinct keys: 1814\n",
      "Number of profiles with no fewer than  5  (latest occupation) occurrences: 45340\n",
      "Percentage of profiles with no fewer than  5  (latest occupation) occurrences: 0.421250185818344\n"
     ]
    }
   ],
   "source": [
    "modelY_cluster_list = list()\n",
    "for i in range(len(modelY.wv.vocab.keys())):\n",
    "    modelY_cluster_list.append(list(modelY.wv.vocab.keys())[i])\n",
    "print('Number of distinct keys:', len(modelY_cluster_list))\n",
    "\n",
    "# Check the number of latest occupations with no fewer than min_count = 5 occurrences\n",
    "y_in_list = 0\n",
    "for i in df['latest occupation']:\n",
    "    i=i.translate(str.maketrans('', '', string.punctuation))\n",
    "    i=i.replace(\" \", \"\")\n",
    "    i=i.lower()\n",
    "    if i in modelY_cluster_list:\n",
    "        y_in_list += 1\n",
    "print('Number of profiles with no fewer than ', min_count, ' (latest occupation) occurrences:', y_in_list)\n",
    "print('Percentage of profiles with no fewer than ', min_count, ' (latest occupation) occurrences:', y_in_list/len(rows_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label array dimensions:  (107632, 1)\n"
     ]
    }
   ],
   "source": [
    "# Build another model with min_count = 1\n",
    "modelY2 = Word2Vec(all_sentencesY, \n",
    "                 min_count=1,   # Ignore words that appear less than this\n",
    "                 size=200,      # Dimensionality of word embeddings\n",
    "                 workers=16,     # Number of processors (parallelisation)\n",
    "                 window=5,      # Context window for words during training\n",
    "                 iter=30)       # Number of epochs training over corpus\n",
    "\n",
    "y_class_array = np.zeros((len(df), 1))\n",
    "y_class_array -= 1\n",
    "print('Label array dimensions: ', y_class_array.shape)\n",
    "modelY2_key_list = list(modelY2.wv.vocab.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill the array\n",
    "# Assign multiclass labels (0 - 49)\n",
    "random.seed(2)\n",
    "df_occup = pd.DataFrame(data = df['latest occupation'])\n",
    "df_occup.columns = [''] * len(df_occup.columns)\n",
    "for index,row in df_occup.iterrows():\n",
    "    person=pd.Series.to_string(row)\n",
    "    person=person.translate(str.maketrans('', '', string.punctuation))\n",
    "    person=person.replace(\" \", \"\")\n",
    "    person=person.lower()\n",
    "    if person in modelY_cluster_list:\n",
    "        i = modelY_cluster_list.index(person)\n",
    "        y_class_array[index] = modelYvec_cluster[i]\n",
    "    else:\n",
    "        for j in range(10):\n",
    "            try:\n",
    "                person_similar = modelY2.most_similar(person)[j][0]\n",
    "            except KeyError:\n",
    "                person_similar = ''\n",
    "            if person_similar in modelY_cluster_list:\n",
    "                i = modelY_cluster_list.index(person_similar)\n",
    "                y_class_array[index] = modelYvec_cluster[i]\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of latest occupations with labels:  0.5614315445220752\n",
      "\n",
      "\n",
      "Label array:  [[14.]\n",
      " [39.]\n",
      " [-1.]\n",
      " ...\n",
      " [14.]\n",
      " [-1.]\n",
      " [-1.]]\n"
     ]
    }
   ],
   "source": [
    "n_neg1 = list(y_class_array).count([-1.])\n",
    "print('Percentage of latest occupations with labels: ',1 - n_neg1/len(df))\n",
    "print('\\n')\n",
    "print('Label array: ', y_class_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaned feature array:  [[ 0.  0.  0. ...  0.  0. 16.]\n",
      " [ 2.  3.  0. ...  0.  0. 21.]\n",
      " [ 0.  0.  0. ...  0.  0. 18.]\n",
      " ...\n",
      " [ 0.  1.  0. ...  0.  0. 13.]\n",
      " [ 0.  1.  0. ...  0.  0. 32.]\n",
      " [ 1.  2.  0. ...  0.  0. 27.]]\n",
      "\n",
      "\n",
      "Cleaned label array:  ['14' '39' '19' ... '11' '12' '14']\n",
      "\n",
      "\n",
      "Number of rows in feature array:  60428\n",
      "\n",
      "\n",
      "Number of rows in label array:  60428\n"
     ]
    }
   ],
   "source": [
    "# Convert int labels to str\n",
    "y_class_array_str = []\n",
    "for i in range(len(y_class_array)):\n",
    "    y_class_array_str.append(str(int(list(y_class_array[i])[0])))\n",
    "\n",
    "# Remove profiles with label = '-1' (Caution)\n",
    "df_noneg1 = pd.DataFrame(feature_array)\n",
    "df_noneg1['Latest Occupation'] = y_class_array_str\n",
    "df_noneg = df_noneg1[df_noneg1['Latest Occupation'] != '-1']\n",
    "feature_array_noneg = np.array(df_noneg.iloc[:,:len(feature_array[0,:])])\n",
    "y_class_array_noneg = np.array(df_noneg.iloc[:,len(feature_array[0,:])])\n",
    "print('Cleaned feature array: ', feature_array_noneg)\n",
    "print('\\n')\n",
    "print('Cleaned label array: ', y_class_array_noneg)\n",
    "print('\\n')\n",
    "print('Number of rows in feature array: ', len(feature_array_noneg))\n",
    "print('\\n')\n",
    "print('Number of rows in label array: ', len(y_class_array_noneg))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model buildup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "random.seed(2)\n",
    "# Train-test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(feature_array_noneg, y_class_array_noneg, random_state = 1, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline prediction for all the test set:  35\n",
      "Baseline test accuracy:  0.043024987588945886\n"
     ]
    }
   ],
   "source": [
    "# Baseline model (always predict the label with the most training set occurrences)\n",
    "import statistics \n",
    "from statistics import mode\n",
    "base_pred = mode(y_train)\n",
    "print('Baseline prediction for all the test set: ', base_pred)\n",
    "\n",
    "s = 0\n",
    "for i in range(len(y_test)):\n",
    "    if int(y_test[i]) == int(base_pred):\n",
    "        s += 1\n",
    "print('Baseline test accuracy: ', s/len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'max_depth')"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4VPX5/vH3k4QECHvCHiDIDmGPqMUdtKAipWArVX9qrba2tLbaqv1aW5duVq11t7ZabbW1Ci64IC644R7CGtbIGhAIENYAIcnz+2MOmsZAAsnkTJL7dV1zZebMOTP3sMyds32OuTsiIiJHKy7sACIiUrepSEREpFpUJCIiUi0qEhERqRYViYiIVIuKREREqkVFIiIi1aIiERGRalGRiIhItSSEHaA2pKamenp6etgxRETqjNTUVGbOnDnT3cdUNm+DKJL09HSysrLCjiEiUqeYWWpV5tOmLRERqRYViYiIVIuKREREqkVFIiIi1aIiERGRalGRiIhItahIRESkWhrEeSR1TVFxKYVFxRQWlVBYVMye/SXsKSqmMPi5t6iEE3qk0C0lOeyoIiIqkuooLXUKD5RQuD/ypb8n+PLfs/9/f5YthMKiYvYURZY5WApfPo48f6DEK33v1GZJvP6zk2mdnFgLn1RE5NBUJIfx+1eWsGbrnkOWwt4DJVV+rTiD5MQEmibF0zQxgaaJ8SQnJtA6OZG01sHjpMjPyC2B5KRyP4Pl83ft56JHPubX03O4d/LQKP4JiIhUTkVyGMs37eLz7ftomhT50k9tlhT5kk9KIPkQX/ZlSyA58cuCSEqIw8xqJFePts34yem9uPP15YzN6MBZAzvWyOuKiBwNFclhPHbpiLAjHNIPTu3Ba4s38avnFzGiextSmyWFHUlEGigdtVVHNYqP485vDWb3vmJufH4R7pXvVxERiYaoFomZjTGzZWaWa2bXV/B8kpn9N3j+YzNLD6ZfYGbzytxKzWxIuWWnm9miaOaPdb3bN+dnZ/RmxqKNvLjg87DjiEgDFbUiMbN44H5gLNAfmGxm/cvNdhlQ4O49gbuA2wDc/Ul3H+LuQ4CLgNXuPq/Ma38T2B2t7HXJFScfw9Curfj1C4vYvGtf2HFEpAGK5hrJCCDX3Ve6exHwFDC+3DzjgceD+1OBUfbVPdKTgf8cfGBmzYCrgd9GJXUdEx9n3HHeYPYWlfB/zy7UJi4RqXXRLJLOwLoyj/OCaRXO4+7FwA4gpdw836ZMkQC3AncChTUZti7r0bYZv/h6H95Ysplns9eHHUdEGphoFklFx7qW/3X5sPOY2XFAobsvCh4PAXq6+3OVvrnZFWaWZWZZ+fn5RxC7brp0ZHeOTW/NTS/msHGHNnGJSO2JZpHkAV3KPE4DNhxqHjNLAFoC28o8fz7/uzZyAjDczFYDs4HeZvZ2RW/u7g+7e6a7Z7Zt27YaH6NuiI8zbp80mOIS57ppC7SJS0RqTTSL5FOgl5l1N7NEIqUwvdw804GLg/uTgFkefAOaWRxwHpF9KwC4+4Pu3snd04ETgeXufmoUP0Odkp6azPVj+/LO8nyezlpX+QIiIjUgakUS7POYAswElgBPu3uOmd1iZucGsz0CpJhZLpEd6GUPET4ZyHP3ldHKWB9ddHw3TjgmhVtfWsL67XvDjiMiDYA1hE0gmZmZnpWVFXaMWrNuWyFj/vIuQ7u25l+XjaixoVlEpGExsznunlnZfDqzvR7q0qYp/3d2P2bnbuHJj9eGHUdE6jkVST31nRFdOalXKr9/ZQlrt+pIaRGJHhVJPWVm3DZxEPFm/GLqfEpL6/8mTBEJh4qkHuvUqgk3ntOfj1dt4/EPV4cdR0TqKRVJPXdeZhqn9WnLba8uZdWWPWHHEZF6SEVSz5kZf5w4iMT4OH7+zHxKtIlLRGqYiqQBaN+iMTePH8CcNQU8OntV2HFEpJ5RkTQQ3xjSmTP6t+f215aRu3lX2HFEpB5RkTQQZsbvJwwkOTGea55ZQHFJadiRRKSeUJE0IG2bJ3HL+Azmr9vOw+9p5BkRqRkqkgbmnEEdOWtgB/7y+gqWbdQmLhGpPhVJA2Nm3Do+g+aNE7jmmXkc0CYuEakmFUkDlNIsid9NyGDR+p088NZnYccRkTpORdJAjcnoyPghnbh31gpyNuwIO46I1GEqkgbs5nMH0Do5kWuenk9RsTZxicjRUZE0YK2aJvKHCQNZunEX985aEXYcEamjVCQN3Oj+7Zk4LI0H3v6M+eu2hx1HROogFYnw63H9adssiZ8/M599B0rCjiMidYyKRGjZpBF/nDiQFZt385c3tIlLRI6MikQAOLVPO84/tgsPv/sZ2WsLwo4jInWIikS+cMPZ/ejYsgk/f1qbuESk6lQk8oXmjRvxp0mDWLllD7fPXBZ2HBGpI1Qk8j9G9kzlouO78ej7q/hk1baw44hIHaAika+4fmxfurRuyi+mzqewqDjsOCIS41Qk8hXJSQncPmkQa7YWctuMpWHHEZEYpyKRCh13TAqXjkzn8Q/X8EHulrDjiEgMU5HIIV379b50T03mF1MXsHu/NnGJSMVUJHJITRLjueO8QWzYsZffvbwk7DgiEqNUJHJYw7u14fKTjuE/n6zl3eX5YccRkRikIpFKXX1Gb3q0Tea6aQvYue9A2HFEJMaoSKRSjRvFc+e3hrBp5z5ufXFx2HFEJMaoSKRKhnRpxZWn9uCZOXnMWrop7DgiEkNUJFJlPxnVi74dmnP9tIVsLywKO46IxIioFomZjTGzZWaWa2bXV/B8kpn9N3j+YzNLD6ZfYGbzytxKzWyImTU1s5fNbKmZ5ZjZH6OZX/5XUkI8d5w3mG17irhZm7hEJBC1IjGzeOB+YCzQH5hsZv3LzXYZUODuPYG7gNsA3P1Jdx/i7kOAi4DV7j4vWOYOd+8LDAVGmtnYaH0G+aqMzi350Wk9eW7uembmbAw7jojEgGiukYwAct19pbsXAU8B48vNMx54PLg/FRhlZlZunsnAfwDcvdDd3wruFwHZQFqU8sshTDm9J/07tuCG5xaybY82cYk0dNEsks7AujKP84JpFc7j7sXADiCl3DzfJiiSssysFTAOeLOG8koVNYqP485vDWbH3gPc+MKisOOISMiiWSTl1ywA/EjmMbPjgEJ3/59vKzNLIFIu97j7ygrf3OwKM8sys6z8fJ1IV9P6dWzBVaN68fKCz3lpwYaw44hIiKJZJHlAlzKP04Dy3zhfzBOUQ0ug7EUwzqeCtRHgYWCFu//lUG/u7g+7e6a7Z7Zt2/Yo4ktlfnBKDwalteTG5xeRv2t/2HFEJCTRLJJPgV5m1t3MEomUwvRy80wHLg7uTwJmubsDmFkccB6RfStfMLPfEimcn0Yxu1RBQnwcd543mD1FJfzq+YUEf3Ui0sBErUiCfR5TgJnAEuBpd88xs1vM7NxgtkeAFDPLBa4Gyh4ifDKQV3bTlZmlATcQOQosOzg0+HvR+gxSuV7tm3PNGb2ZmbOJ6fO1iUukIbKG8FtkZmamZ2VlhR2j3iopdc576AM+y9/Daz87mfYtGocdSURqgJnNcffMyubTme1SbfFxxh3nDWbfgRK+/dcPufuNFazYtCvsWCJSS7RGIjVm1tJNPPj2Z2StKcAderRNZmxGR8ZkdGBApxZ89RQhEYllVV0jUZFIjdu8cx8zczYyY9FGPl61jZJSp0ubJl+UypC0VsTFqVREYp2KpAwVSXi27Sni9cWRUnk/dwsHSpwOLRozJqMDYzI6cGx6G+JVKiIxSUVShookNuzYe4BZSzcxY+FG3lmez/7iUlKbJXLmgA6MzejA8cek0Cheu+1EYoWKpAwVSezZs7+Yt5ZtZsaijby1dDOFRSW0atqI0f3aMzajAyf2SiUpIT7smCINmoqkDBVJbNt3oIR3l+fz6qKNvL5kE7v2FdMsKYFR/doxNqMDp/RuR5NElYpIbatqkSTURhiRw2ncKJ4zB3TgzAEdKCou5f3PtvDqwo28tngjL8zbQJNG8Zzapy1jMjpwet92NG/cKOzIIlKG1kgkZhWXlPLJqm3MWLSRV3M2kr9rP4nxcZzUK5UxGR04o397WjVNDDumSL2lTVtlqEjqvtJSJ3ttAa8s3MjMnI2s376XhDjjhB4pjM3oyJkD2pPaLCnsmCL1ioqkDBVJ/eLuLMjbEVlTWfQ5q7cWEmdwbHobxmZ0YExGRzq01DAtItWlIilDRVJ/uTtLN+5ixsLPmbFoIys27wZgaNdWjM3owNiMjnRp0zTklCJ1k4qkDBVJw5G7eTevLoqUSs6GnQBkdG7B2QM7MWFoZ62piBwBFUkZKpKGae3WQl7N+ZxXFm5k3rrtmMGJPVOZOCyNrw/ooEOKRSqhIilDRSKrt+zh2ew8pmWvZ/32vTRLSuCsgR2YOCyNEd3baEBJkQqoSMpQkchBpaXOx6u2MS07j1cWfk5hUQld2jThm0PTmDgsja4p2p8icpCKpAwViVSksKiYVxdtZFp2Hh98thV3GJHehonDO3PWwI468VEaPBVJGSoSqcz67Xt5fu56ps3JY+WWPTRuFMfXB0Q2fY3smaoRiqVBUpGUoSKRqnJ35q7bzrQ5ebw4fwM79xXToUVjvjG0M5OGd6Znu+ZhRxSpNSqSMlQkcjT2HSjhzSWbmZadxzvL8ykpdQantWTi8DTGDepE62QNzyL1m4qkDBWJVNfmXfuYPm8DU+fksXTjLhrFG6P6tmfS8DRO6dNW11GReklFUoaKRGpSzoYdTJuznhfmrWfrniJSkhMZP6QzE4d3ZkCnlmHHE6kxKpIyVCQSDQdKSnlnWT7TsvN4c8lmikpK6duhOROHpTF+aCfaNddZ9FK31WiRmNk04FFghruX1kC+WqUikWjbXljEi/M3MDV7PfPXbSc+zji5VyoTh6cxul97GjfSWfRS99R0kYwGLgWOB54BHnP3pdVOWUtUJFKbcjfvYlr2ep7LXs/Gnfto0TiBcwZ3YuKwNIZ1baWz6KXOiMqmLTNrCUwGbgDWAX8DnnD3A0cbtDaoSCQMJaXOB59tYdqcPF7N2ci+A6Uck5rMxOFpTBjamU6tmoQdUeSwarxIzCwFuBC4CNgAPAmcCAx091OPPmr0qUgkbLv2HWDGwo1MnZPHJ6u3YQZf65HCxSekc+aADmHHE6lQTW/aehboC/yLyGatz8s8l1WVNwqTikRiydqthTw7N49p2Xms27aX84/twm/GDdBoxBJzarpITnf3WTWSLAQqEolFB0pKuev15Tzw9mf0bt+M+78zjF7tdea8xI6qFklVz6LqZ2atyrx4azP74VGnExEaxcdx7Zi+PP7dEWzdXcS4+2bzdNY6GsIh+VK/VLVILnf37QcfuHsBcHl0Iok0LKf0bsuMq05iaJfWXDt1AVc/PZ/d+4vDjiVSZVUtkjgrc8yimcUDGmhIpIa0a9GYJ753HD8b3ZsX5q3n3Htnk7NhR9ixRKqkqkUyE3jazEaZ2enAf4BXoxdLpOGJjzOuGt2LJ793PLv3FzPhgQ/410drtKlLYl5Vi+Q6YBZwJfAj4E3g2miFEmnITuiRwoyrTuKEY1K48flF/Ojf2ezYG9OnakkDV6UicfdSd3/Q3Se5+0R3/6u7l1S2nJmNMbNlZpZrZtdX8HySmf03eP5jM0sPpl9gZvPK3ErNbEjw3HAzWxgsc0/ZTW4i9UVKsyT+ccmxXD+2LzNzNnHOve8xf932yhcUCUGVisTMepnZVDNbbGYrD94qWSYeuB8YC/QHJptZ/3KzXQYUuHtP4C7gNgB3f9Ldh7j7ECInQK5293nBMg8CVwC9gtuYKn1SkTomLs74wSk9ePr7J1BaCpMe+oC/v7dSm7ok5lR109Y/iHyBFwOnAf8kcnLi4YwAct19pbsXAU8B48vNMx54PLg/FRhVwRrGZCL7ZDCzjkALd//QI/+b/gl8o4qfQaROGt6tNS//5ERO69OO3768hMv/mUXBnqKwY4l8oapF0sTd3yRyAuMad78JOL2SZToTGY/roLxgWoXzuHsxsANIKTfPtwmKJJg/r5LXFKl3WjVN5K8XDeemcf15d/kWzrrnPbJWbws7lghQ9SLZZ2ZxwAozm2JmE4B2lSxT0b6L8uvkh53HzI4DCt190RG85sFlrzCzLDPLys/PrySqSOwzMy4Z2Z1pV36NxIQ4vv3wR9z/Vi6lpdrUJeGqapH8FGgK/AQYTmTwxosrWSYP6FLmcRqRwR4rnMfMEoCWQNlfs87ny7WRg/OnVfKaALj7w+6e6e6Zbdu2rSSqSN0xMK0lL/34RMZmdOD2mcu4+B+fkL9rf9ixpAGrtEiCnebfcvfd7p7n7pcGR259VMminwK9zKy7mSUSKYXp5eaZzpeFNAmYFez7IFgDOo/IvhUAgsEid5nZ8cG+lP8HvFD5xxSpX5o3bsS9k4fyh28O5JNV2zjrnvf4IHdL2LGkgaq0SILDfIcf6WG2wT6PKUROZlwCPO3uOWZ2i5mdG8z2CJBiZrnA1UDZQ4RPBvLcvfzRYVcCfwdygc+AGUeSS6S+MDMmj+jKC1NG0qJxAhc88jF/fn05JdrUJbWsqqP/3knkUNtngD0Hp7v7s9GLVnM0+q/Ud4VFxfz6hRymzsnjuO5tuPv8oXRoqWvGS/XU9Oi/bYCtRI7UGhfczjn6eCJSk5omJnDHeYO587zBLFy/g7PueY+3lm0OO5Y0EEd0qd26Smsk0pDkbt7NlH9ns3TjLr5/yjH8/Mw+NIqv6u+MIl+q6hpJQhVf7B9UcJitu3/3KLKJSBT1bNeM5380klteWsxf31nJp6u2cc/koaS1bhp2tGo5+EuvRkWKPVUqEuClMvcbAxM4xGG3IhK+xo3i+f2EgZxwTAq/fHYhZ98zm9snDapT14cvKi4lZ8MO5qwp4NPV25izpoBG8XH85dtDOO6Y8uctS5iOatNWcGjuG+5e2dntMUGbtqQhW7N1D1P+PZeF63dw6ch0rh/bl6SE2Ls+/I69B8heW0DW6m1krS5gft529h0oBaBrm6ZkdmvN3HXbWbutkBvO6selI9O1dhJlNbppqwK9gK5HuayI1KJuKclMvfIEbpuxjEffX8Wnq7dx3+RhpKcmh5bJ3ckr2EvWmkhpZK0uYPnmXbhHrssyoFMLvjOiG5nprcns1pp2LSJHoO3cd4Cr/zufW15azPy87fzhmwNpmni0X2NSU6p6+O8u/ncfyUbgl+4+LVrBapLWSEQiXsvZyC+mLqCk1PnDNwcybnCnWnnf4pJSlny+68viWLONTTsjZ+M3S0pgWLdIYWSmt2ZIl1aHLYfSUueBt3O58/Xl9GnfnL9eNJxuKeGVYn1W1TUSHbUl0sCs376XH/87m+y125k8oiu/Gdefxo1qdlPX7v3FzF1bwKerC5izZhtz126nsChyCaPOrZowvFtrjk1vzfBubejToTnxcUe+iertZZu56ql5uDt3nz+U0/pWNvyfHKkaLZJgkMZZ7r4jeNwKONXdn6920lqgIhH5XwdKSvnz68t58O3P6NuhOfd9Zyg92zU/6tfbsH0vWWsKmLN6G5+uLmDpxp2UOsQZ9O3QIlIa6W3I7NaaTq2a1NjnWLu1kO8/MYelG3fys9G9mXJaT+KOopSkYjVdJPOCi0yVnTbX3YdWI2OtUZGIVOztZZu55un5FBaVcOs3Mpg0PK3SZUpKnWUbv9xMNWdNAeu37wWgaWI8Q7u2Yni3NhwbbKZq3rhRVD/D3qIS/u+5hTw3dz2j+7Xjzm8NoWWT6L5nQ1HTRbLA3QeVm7bQ3QdWI2OtUZGIHNqmnfu46qm5fLRyG98c1plbx2eQnPTlPorComLmrd1O1poCstYUMHdNAbv2FwPQvkUSmd3aBDvF29CvY3MSQjj50d3554druPWlxXRp05SHLhxOnw5Hv4YlETVdJI8C24lcOteBHwOt3f2SauasFSoSkcMrKXXunbWCu99cQffUZK48pccXO8dzNuykpNQxg97tmkdKIyiOtNZNYuoQ3E9Xb+OHT2azZ38xf5o0iHMG1c7BBPVVTRdJMnAjMDqY9BrwO3ffc+ilYoeKRKRqPvhsCz99ah6bd+2ncaM4Bqe1CoqjDcO6tq4Tm4w27dzHD5/MZs6aAi4/qTvXjekbylpSfaCjtspQkYhU3Y69B1i3rZDe7ZuTmFA3v4CLikv57cuL+eeHazj+mDbc951hpDZLCjtWnVOjo/+a2evBkVoHH7c2s5nVCSgisallk0ZkdG5ZZ0sEIDEhjlvGZ3DHeYOZu3Y74+6dzbx128OOVW9V9V9Kqrt/8bfg7gVUfs12EZFQTRqexrQrv0Z8nPGthz7kqU/Whh2pXqpqkZSa2RdDophZOhWMBiwiEmsyOrfkxSknctwxbbj+2YX88tkF7C8uCTtWvVLVQWpuAGab2TvB45OBK6ITSUSkZrVOTuSxS0dw52vLeODtz1i8YScPXji8Rk+ObMiqtEbi7q8CmcAy4L/ANcDeKOYSEalR8XHGtWP68tCFw/ksfw/j7p3NB59tCTtWvVDVne3fA94kUiDXAP8CbopeLBGR6BiT0YHnfzSSVk0bcdEjn/C3d1fSEI5ejaaq7iO5CjgWWOPupwFDgfyopRIRiaKe7ZrxwpQTOaNfe373yhKm/Gcue4Kz9eXIVbVI9rn7PgAzS3L3pUCf6MUSEYmuZkkJPHjhMK4b05cZCz9nwgPvs2pLnTjHOuZUtUjygvNIngdeN7MX0KV2RaSOMzOuPLUH//zuceTv2s+5987mjcWbwo5V5xzxme1mdgrQEnjV3YuikqqG6cx2EalMXkEhP3hiDovW7+Qno3rx01G9GvyQ9DV6ZntZ7v6Ou0+vKyUiIlIVaa2bMvUHX2PS8DTueXMFlz3+KTsKD4Qdq06ou2MgiIjUsMaN4rl90iB++40MZuduYdx9s1ny+c6wY8U8FYmISBlmxoXHd+OpK05gf3EJEx54nxfmrQ87VkxTkYiIVGB4t9a8+OMTGdS5FVc9NY+bX8zhQElp2LFikopEROQQ2jVvzJOXH8elI9P5x/urueBvH7N5176wY8UcFYmIyGE0io/jN+MGcPf5Q1iwPjIk/Zw1BWHHiikqEhGRKhg/pDPP/XAkSQnxnP/whzzx0RoNrRJQkYiIVFG/ji14ccqJjOyZyq+eX8S1Uxew74CGpFeRiIgcgZZNG/Hoxcfyk1G9eGZOHuc99CF5BYVhxwqVikRE5AjFxRlXn9Gbv/+/TFZv2cOEBz5o0CcvRrVIzGyMmS0zs1wzu76C55PM7L/B8x8HV148+NwgM/vQzHLMbKGZNQ6mTw4eLzCzV80sNZqfQUTkUEb3b8+/Lz+erbv3c9cby8OOE5qoFYmZxQP3A2OB/sBkM+tfbrbLgAJ37wncBdwWLJsAPAH8wN0HAKcCB4LpdwOnufsgYAEwJVqfQUSkMgPTWjJ5RFf+9dEalm/aFXacUERzjWQEkOvuK4NxuZ4CxpebZzzweHB/KjDKzAw4E1jg7vMB3H2ru5cAFtySg/laoFGIRSRk15zZh+TEeG55cXGDPJIrmkXSGVhX5nFeMK3Cedy9GNgBpAC9ATezmWaWbWbXBvMcAK4EFhIpkP7AI1H8DCIilWqTnMjVZ/Rmdu4WXmuAw9BHs0gqGn+5fFUfap4E4ETgguDnBDMbZWaNiBTJUKATkU1bv6zwzc2uMLMsM8vKz9fFHEUkui48vhu92zfjty8vbnCHBEezSPKALmUep/HVzVBfzBPs/2gJbAumv+PuW9y9EHgFGAYMAXD3zzyy/vg08LWK3tzdH3b3THfPbNu2bc19KhGRCiQEZ8Cv27aXR2avCjtOrYpmkXwK9DKz7maWCJwPTC83z3Tg4uD+JGBWUBAzgUFm1jQomFOAxcB6oL+ZHWyGM4AlUfwMIiJVNrJnKmMGdOC+Wbl8vmNv2HFqTdSKJNjnMYVIKSwBnnb3HDO7xczODWZ7BEgxs1zgauD6YNkC4M9EymgekO3uL7v7BuBm4F0zW0BkDeX30foMIiJH6oaz+1Hizh9nLA07Sq054kvt1kW61K6I1KY7X1vGvbNymfqDE8hMbxN2nKMWtUvtiojI4V15ag86tmzMTS/mUFJa/39ZV5GIiNSwpokJXD+2L4vW7+SZrHWVL1DHqUhERKLg3MGdODa9NbfPXMaOvfV7HC4ViYhIFJgZvxk3gG2FRdz9xoqw40SVikREJEoyOrfk/GO78s8PV5O7uf6Ow6UiERGJop+f2ZsmifHcXI/H4VKRiIhEUUqzJH42ujfvrdjCG0s2hx0nKlQkIiJRdtEJ3ejVLjIO1/7i+jcOl4pERCTKGsXH8etx/VmztbBejsOlIhERqQUn9WrLGf3bc9+sXDbt3Bd2nBqlIhERqSU3nt2f4tL6Nw6XikREpJZ0TWnK5Sd157m565mzpiDsODVGRSIiUot+eGpP2rdI4uYXcyitJ+NwqUhERGpRclICvxzbjwV5O5g6Jy/sODVCRSIiUsvGD+nE8G6t+dPMpezcV/fH4VKRiIjUMjPjpnED2LqniHvfrPvjcKlIRERCMDCtJd8a3oV/vL+a3M27w45TLSoSEZGQ/GJMH5o0iufWl+r2OFwqEhGRkKQ2S+Kq0b14Z3k+s5bW3XG4VCQiIiG6+Gvp9GibzK0v1d1xuFQkIiIhiozDNYDVWwv5x/urw45zVFQkIiIhO6V3W0b3a8e9b65gcx0ch0tFIiISA351dn8OlDi3vbos7ChHTEUiIhID0lOT+e6J3ZmWncfctXVrHC4ViYhIjJhyek/aNU/ipul1axwuFYmISIxolpTA9WP7Mj9vB9Oy6844XCoSEZEY8o0hnRnatRW3vbqMXXVkHC4ViYhIDImLi4zDtWX3fu6blRt2nCpRkYiIxJjBXVpx3vA0Hn1/FSvzY38cLhWJiEgM+sWYPiQlxPPbl5eEHaVSKhIRkRjUrnljfjKqJ7OWbuatGB+HS0UiIhKjLvlad45JjYzDVVRcGnacQ1KRiIjEqMSEOG4c15+VW/bw2Aerwo5zSFEtEjMbY2bLzCzXzK6v4PkkM/tv8PzHZpZe5rlBZvahmeWY2UIzaxxMTzSzh80Y/dQtAAAKcklEQVRsuZktNbOJ0fwMIiJhOq1PO07v24573sxl867YHIcrakViZvHA/cBYoD8w2cz6l5vtMqDA3XsCdwG3BcsmAE8AP3D3AcCpwMEDqm8ANrt77+B134nWZxARiQU3ntOf/cUl3B6j43BFc41kBJDr7ivdvQh4Chhfbp7xwOPB/anAKDMz4ExggbvPB3D3re5+cKD+7wJ/CKaXuvuWKH4GEZHQdU9N5rsju/PMnDzmrdsedpyviGaRdAbWlXmcF0yrcB53LwZ2AClAb8DNbKaZZZvZtQBm1ipY7tZg+jNm1j6Kn0FEJCZMOb0nqc1icxyuaBaJVTCt/Kc/1DwJwInABcHPCWY2KpieBrzv7sOAD4E7KnxzsyvMLMvMsvLz84/yI4iIxIbmjRtx3Zg+zFu3nefmrg87zv+IZpHkAV3KPE4DNhxqnmC/SEtgWzD9HXff4u6FwCvAMGArUAg8Fyz/TDD9K9z9YXfPdPfMtm3b1swnEhEJ0cRhaQzu0oo/vrqU3fuLw47zhWgWyadALzPrbmaJwPnA9HLzTAcuDu5PAma5uwMzgUFm1jQomFOAxcFzLxLZ+Q4wClgcxc8gIhIzIuNw9Sd/V2yNwxW1Ign2eUwhUgpLgKfdPcfMbjGzc4PZHgFSzCwXuBq4Pli2APgzkTKaB2S7+8vBMtcBN5nZAuAi4JpofQYRkVgztGtrJg5L49HZq1i9ZU/YcQCwyC/59VtmZqZnZWWFHUNEpEZs3rmP0+54mxN6pPD3i4+N2vuY2Rx3z6xsPp3ZLiJSx7Rr0Zgfj+rFG0s28/ay8MfhUpGIiNRBl45MJz2lKbfEwDhcKhIRkTooKSGeX4/rz8r8Pfzzw9WhZlGRiIjUUaf3bc+pfdpy9xsryN+1P7QcKhIRkTrsxnP6s/dACXfMDG8cLhWJiEgd1qNtMy4dmc7Tc9axMG9HKBlUJCIiddyPR/UiJTmRm17MIYxTOlQkIiJ1XIvGjbj2632Zs6aAF+aVH4kq+lQkIiL1wKThaQxKa8kfZixhTy2Pw6UiERGpB+LijN+MG8Cmnfu5/63aHYdLRSIiUk8M79aabw7tzN/fW8WarbU3DpeKRESkHrlubF8S4o3fvryk1t5TRSIiUo+0b9GYKaf35PXFm3hvRe1c1E9FIiJSz1x2Yne6pTTl5hcXc6Ak+uNwqUhEROqZpIR4fnV2f1KSEykoLIr6+yVE/R1ERKTWje7XjtH92mFmUX8vFYmISD1UGwVykDZtiYhItahIRESkWlQkIiJSLSoSERGpFhWJiIhUi4pERESqRUUiIiLVYmFcTau2mVk+sOYoF08FttRgnJqiXEdGuY6Mch2Z+phrC4C7j6lsxgZRJNVhZlnunhl2jvKU68go15FRriPT0HNp05aIiFSLikRERKpFRVK5h8MOcAjKdWSU68go15Fp0Lm0j0RERKpFayQiIlItKpJDMLOfmVmOmS0ys/+YWeOwMwGY2VVBphwz+2nIWR41s81mtqjMtDZm9rqZrQh+to6RXOcFf2alZhbK0TWHyHW7mS01swVm9pyZtYqRXLcGmeaZ2Wtm1ikWcpV57udm5maWGgu5zOwmM1sf/HnNM7OzYiFXMP3HZrYs+Pf/p2i8t4qkAmbWGfgJkOnuGUA8cH64qcDMMoDLgRHAYOAcM+sVYqTHgPLHmF8PvOnuvYA3g8e17TG+mmsR8E3g3VpP86XH+Gqu14EMdx8ELAd+WduhqDjX7e4+yN2HAC8Bv671VBXnwsy6AGcAa2s7UOAxKsgF3OXuQ4LbK7WcCSrIZWanAeOBQe4+ALgjGm+sIjm0BKCJmSUATYENIecB6Ad85O6F7l4MvANMCCuMu78LbCs3eTzweHD/ceAbtRqKinO5+xJ3X1bbWcplqCjXa8HfJcBHQFqM5NpZ5mEyUOs7Uw/x7wvgLuBaQsgEh80VqkPkuhL4o7vvD+bZHI33VpFUwN3XE2nutcDnwA53fy3cVEDkt+qTzSzFzJoCZwFdQs5UXnt3/xwg+Nku5Dx1yXeBGWGHOMjMfmdm64ALCGeN5CvM7FxgvbvPDztLBaYEmwMfDWOT7iH0Bk4ys4/N7B0zOzYab6IiqUDwj2A80B3oBCSb2YXhpor8Vg3cRmRzyKvAfKD4sAtJnWBmNxD5u3wy7CwHufsN7t6FSKYpYecJfnm6gRgptXIeBHoAQ4j88nlnuHG+kAC0Bo4HfgE8bVG4Bq+KpGKjgVXunu/uB4Bnga+FnAkAd3/E3Ye5+8lEVmNXhJ2pnE1m1hEg+BmVVen6xMwuBs4BLvDYPB7/38DEsEMQ+aLuDsw3s9VENgNmm1mHUFMB7r7J3UvcvRT4G5H9mLEgD3jWIz4BSomMv1WjVCQVWwscb2ZNg/YeBSwJORMAZtYu+NmVyM7j/4Sb6CumAxcH9y8GXggxS8wzszHAdcC57l4Ydp6Dyh3EcS6wNKwsB7n7Qndv5+7p7p5O5EtymLtvDDnawV+aDppAZDN0LHgeOB3AzHoDiURjcEl3162CG3Azkf88i4B/AUlhZwpyvQcsJrJZa1TIWf5DZDX+AJH/1JcBKUSO1loR/GwTI7kmBPf3A5uAmTGSKxdYB8wLbg/FSK5pwb/9BcCLQOdYyFXu+dVAaizkCr4jFgZ/XtOBjjGSKxF4Ivi7zAZOj8Z768x2ERGpFm3aEhGRalGRiIhItahIRESkWlQkIiJSLSoSERGpFhWJiIhUi4pEJEaY2eqjHRbdzC4pO9R7dV5L5EipSETqh0uIjAsnUutUJCLlmFl6cLGpvwcXEXvSzEab2fvBBbtGBLcPzGxu8LNPsOzVZvZocH9gsHzTQ7xPSnDRqLlm9lfAyjx3oZl9Elwk6a9mFh9M321md5pZtpm9aWZtzWwSkAk8GczfJHiZHwfzLTSzvtH8M5OGTUUiUrGewN3AIKAv8B3gRODnwP8RGT7nZHcfSmQ02t8Hy/0F6GlmE4B/AN/3Q4+h9RtgdvAa04GuAGbWD/g2MNIjF5YqITKUO0SuDZLt7sOIXI/mN+4+FcgiMujjEHffG8y7JZjvwSC3SFQkhB1AJEatcveFAGaWQ+Sqj25mC4F0oCXweDC4oQONANy91MwuITLm0l/d/f3DvMfJRAbexN1fNrOCYPooYDjwaTDidxO+HEW5FPhvcP8JIiNTH8rB5+YcfB+RaFCRiFRsf5n7pWUelxL5f3Mr8Ja7TzCzdODtMvP3AnZTtX0WFQ12Z8Dj7l6Vy+4ebrC8g5lL0P91iSJt2hI5Oi2B9cH9Sw5ONLOWRDaJnQykBPsvDuVdgk1WZjaWyAWIIDJq8qQylwxoY2bdgufigIOv+R1gdnB/F9C8Gp9H5KipSESOzp+AP5jZ+0B8mel3AQ+4+3Iiw3j/8WAhVOBmIpdOzgbOJHIdHNx9MfAr4DUzW0DkipgHr3exBxhgZnOIXGfilmD6Y8BD5Xa2i9QKDSMvUoeY2W53bxZ2DpGytEYiIiLVojUSkSgzs0uBq8pNft/dfxRGHpGapiIREZFq0aYtERGpFhWJiIhUi4pERESqRUUiIiLVoiIREZFq+f9/9XxwCH/FKAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Decision tree classification (tune max_depth)\n",
    "max_depth_list = [8,9,10,11,12,13,14,15,16]\n",
    "dtree_accuracy_temp = []\n",
    "for max_depth in max_depth_list:\n",
    "    dtree_model_temp = DecisionTreeClassifier(max_depth = max_depth).fit(X_train, y_train)\n",
    "    dtree_accuracy_temp.append(dtree_model_temp.score(X_test, y_test))\n",
    "    \n",
    "plt.plot(max_depth_list, dtree_accuracy_temp)\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('max_depth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision tree test accuracy:  0.0743835843124276\n",
      "\n",
      "\n",
      "Decision tree confusion matrix:\n",
      " [[ 88   6   0 ...   0   0  31]\n",
      " [ 44  26   0 ...   0   0  14]\n",
      " [ 46   3   1 ...   0   0  16]\n",
      " ...\n",
      " [ 42   6   0 ...   0   0  19]\n",
      " [ 30   0   0 ...   0   0  23]\n",
      " [ 54  12   1 ...   1   0 194]]\n",
      "\n",
      "\n",
      "Decision tree important features:\n",
      " [0.01126547 0.02582838 0.01274641 0.02242486 0.00592193 0.00405665\n",
      " 0.00149857 0.00781198 0.00667867 0.00920499 0.00170634 0.04123056\n",
      " 0.00836832 0.0090266  0.00308791 0.01071082 0.01251243 0.01261853\n",
      " 0.00551443 0.00625933 0.00735597 0.00723774 0.01040075 0.00380157\n",
      " 0.00140557 0.02803408 0.02085403 0.00342179 0.0062054  0.00939933\n",
      " 0.01185397 0.01569893 0.00339991 0.00692885 0.02819092 0.01092565\n",
      " 0.02177231 0.00188643 0.02343218 0.00145241 0.00961941 0.12254964\n",
      " 0.00727419 0.02618854 0.00407124 0.0089324  0.00328133 0.\n",
      " 0.00227413 0.01807377 0.0047292  0.00407449 0.01398334 0.02234091\n",
      " 0.00282666 0.0014714  0.00374193 0.00291596 0.00185487 0.00391905\n",
      " 0.00140274 0.0016694  0.00318176 0.00192128 0.         0.01044476\n",
      " 0.00341243 0.         0.         0.00183549 0.02403644 0.01018347\n",
      " 0.0129834  0.00557519 0.00752255 0.         0.00093516 0.\n",
      " 0.         0.00191099 0.         0.         0.         0.00177518\n",
      " 0.         0.01391369 0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.00172371 0.\n",
      " 0.         0.00211802 0.         0.         0.         0.00736026\n",
      " 0.         0.00178125 0.         0.00184422 0.00377737 0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.00823712 0.         0.         0.00442728\n",
      " 0.         0.00364709 0.         0.02731857 0.00374063 0.02450833\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.10055882]\n"
     ]
    }
   ],
   "source": [
    "max_depth = max_depth_list[dtree_accuracy_temp.index(max(dtree_accuracy_temp))]\n",
    "dtree_model = DecisionTreeClassifier(max_depth = max_depth).fit(X_train, y_train)\n",
    "dtree_predictions = dtree_model.predict(X_test)\n",
    "dtree_accuracy = dtree_model.score(X_test, y_test)\n",
    "dtree_cm = confusion_matrix(y_test, dtree_predictions)\n",
    "print('Decision tree test accuracy: ', dtree_accuracy)\n",
    "print('\\n')\n",
    "print('Decision tree confusion matrix:\\n', dtree_cm)\n",
    "print('\\n')\n",
    "print('Decision tree important features:\\n', dtree_model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'log_alpha')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAE0RJREFUeJzt3X+wZ3V93/Hny11YydKAhY0NLGaXwmg2ITFy3Zox4xRIW8gPtliooOlQowHbkJqGVqHMiNVJG0YTkqnEDhUItVYwGJMNtSAER6tJCBdFfggkOwzCCoY1y49IS5eFd/84Z9PL9S73cD977ne/u8/HzJ39ns/3c855f2Zn93XP5/xKVSFJ0lK9bNIFSJKmm0EiSWpikEiSmhgkkqQmBokkqYlBIklqYpBIkpoYJJKkJgaJJKnJykkXsBwOP/zwWrdu3aTLkKSpcfvtt3+7qtYM6btfBMm6deuYnZ2ddBmSNDWSfGNoX6e2JElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNDBJJUpNRgyTJyUnuT7IlyQULfL8qybX997cmWde3H5Dk6iR3Jbk3yYVz1jk0yXVJ7uu/+/ExxyBJenGjBUmSFcBlwCnABuCsJBvmdXsH8HhVHQNcClzSt58BrKqq44DjgXN3hQzwW8ANVfUa4EeBe8cagyRpcWMekWwEtlTVA1W1A7gG2DSvzybg6v7zdcBJSQIUsDrJSuAgYAfwVJLvBd4EXAFQVTuq6okRxyBJWsSYQXIk8PCc5a1924J9qmon8CRwGF2oPA08CjwEfLiqtgNHA9uAq5J8NcnHkqxeaOdJzkkym2R227Zte3BYkqS5xgySLNBWA/tsBJ4DjgDWA+cnORpYCbwO+GhV/Rhd2HzXuReAqrq8qmaqambNmjVLHIIkaTFjBslW4Kg5y2uBR3bXp5/GOgTYDryV7jzIs1X1GPBlYKbvv7Wqbu3Xv44uWCRJEzJmkNwGHJtkfZIDgTOBzfP6bAbO7j+fDtxSVUU3nXViOquBNwD3VdW3gIeTvLpf5yTg6yOOQZK0iJVjbbiqdiY5D7gRWAFcWVX3JPkAMFtVm+lOmn88yRa6I5Ez+9UvA64C7qab/rqqqu7sv/sl4BN9OD0AvH2sMUiSFpfuAGDfNjMzU7Ozs5MuQ5KmRpLbq2pmSF/vbJckNTFIJElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNRg2SJCcnuT/JliQXLPD9qiTX9t/fmmRd335AkquT3JXk3iQXzltvRZKvJrl+zPolSYsbLUiSrAAuA04BNgBnJdkwr9s7gMer6hjgUuCSvv0MYFVVHQccD5y7K2R67wbuHat2SdJwYx6RbAS2VNUDVbUDuAbYNK/PJuDq/vN1wElJAhSwOslK4CBgB/AUQJK1wE8DHxuxdknSQGMGyZHAw3OWt/ZtC/apqp3Ak8BhdKHyNPAo8BDw4ara3q/zm8B7gOdHq1ySNNiYQZIF2mpgn43Ac8ARwHrg/CRHJ/kZ4LGqun3RnSfnJJlNMrtt27aXWLokaagxg2QrcNSc5bXAI7vr009jHQJsB94K3FBVz1bVY8CXgRngjcCpSR6kmyo7Mcl/W2jnVXV5Vc1U1cyaNWv23KgkSS8wZpDcBhybZH2SA4Ezgc3z+mwGzu4/nw7cUlVFN511YjqrgTcA91XVhVW1tqrW9du7pap+bsQxSJIWMVqQ9Oc8zgNupLvC6lNVdU+SDyQ5te92BXBYki3ArwC7LhG+DDgYuJsukK6qqjvHqlWStHTpDgD2bTMzMzU7OzvpMiRpaiS5vapmhvT1znZJUhODRJLUxCCRJDUxSCRJTQwSSVKTQUGS5NNJfjqJwSNJeoGhwfBRurvN/yLJryV5zYg1SZKmyKAgqaqbq+ptwOuAB4GbkvxxkrcnOWDMAiVJe7fBU1VJDgP+OfBO4KvAb9EFy02jVCZJmgorh3RK8nvAa4CPAz9bVY/2X12bxFvGJWk/NihIgI9U1S0LfTH0FnpJ0r5p6NTWDyY5dNdCklck+Zcj1SRJmiJDg+QXquqJXQtV9TjwC+OUJEmaJkOD5GX9u9QBSLICOHCckiRJ02ToOZIbgU8l+c90r8J9F3DDaFVJkqbG0CB5L3Au8C/o3rP+OeBjYxUlSZoeg4Kkqp6nu7v9o+OWI0maNkPvIzkW+I/ABuDlu9qr6uiR6pIkTYmhJ9uvojsa2QmcAPxXupsTJUn7uaFBclBV/RHdO96/UVXvB04cryxJ0rQYerL9mf4R8n+R5Dzgm8D3jVeWJGlaDD0i+WXge4B/BRwP/Bxw9lhFSZKmx6JHJP3Nh/+0qv4t8B3g7aNXJUmaGosekVTVc8Dxc+9slyRpl6HnSL4K/EGS3wWe3tVYVb83SlV7iX//h/fw9UeemnQZkrQkG474Xi7+2R8afT9Dg+RvA3/FC6/UKmCfDhJJ0uKG3tm+X54XWY4kl6RpN/TO9qvojkBeoKp+fo9XJEmaKkOntq6f8/nlwGnAI3u+HEnStBk6tfXpuctJPgncPEpFkqSpMvSGxPmOBV61JwuRJE2noedI/poXniP5Ft07SiRJ+7mhU1t/a+xCJEnTadDUVpLTkhwyZ/nQJP94vLIkSdNi6DmSi6vqyV0LVfUEcPE4JUmSpsnQIFmo39BLhyVJ+7ChQTKb5DeS/N0kRye5FLh9zMIkSdNhaJD8ErADuBb4FPB/gF8cqyhJ0vQYetXW08AFI9ciSZpCQ6/auinJoXOWX5HkxgHrnZzk/iRbknxXECVZleTa/vtbk6zr2w9IcnWSu5Lcm+TCvv2oJJ/v2+5J8u6hA5UkjWPo1Nbh/ZVaAFTV4yzyzvb+zYqXAacAG4CzkmyY1+0dwONVdQxwKXBJ334GsKqqjqN7te+5fcjsBM6vqh8E3gD84gLblCQto6FB8nySv3kkSv+f+nc9DXiejcCWqnqgqnYA1wCb5vXZBFzdf74OOKl/E2MBq5OsBA6iOz/zVFU9WlVfAaiqvwbuBY4cOAZJ0giGXsJ7EfClJF/ol98EnLPIOkcCD89Z3gr8vd31qaqdSZ4EDqMLlU3Ao8D3AP+6qrbPXbEPsx8Dbh04BknSCAYdkVTVDcAMcD/dlVvn01259WIWesf7/KOY3fXZCDwHHAGsB85PcvTfrJQcDHwa+OWqWvBduEnOSTKbZHbbtm2LlCpJWqqhD218J/BuYC1wB935iT/hha/enW8rcNSc5bV89ztMdvXZ2k9jHQJsB94K3FBVzwKPJfkyXZA9kOQAuhD5xIu9M76qLgcuB5iZmVlsGk6StERDz5G8G3g98I2qOoFuSmmxX/NvA45Nsj7JgcCZwOZ5fTYDZ/efTwduqaoCHgJOTGc1XXDd158/uQK4t6p+Y2DtkqQRDQ2SZ6rqGegu2a2q+4BXv9gKVbUTOA+4ke6k+Keq6p4kH0hyat/tCuCwJFuAX+H/36tyGXAwcDddIF1VVXcCbwT+GV3I3NH//NTQwUqS9ryhJ9u39veR/D5wU5LHGfCq3ar6LPDZeW3vm/P5GbpLfeev953dtH+Jhc+rSJImZOid7af1H9+f5PN05zJuGK0qSdLUeMlP8K2qLyzeS5K0v1jqO9slSQIMEklSI4NEktTEIJEkNTFIJElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU1GDZIkJye5P8mWJBcs8P2qJNf239+aZF3ffkCSq5PcleTeJBcO3aYkaXmNFiRJVgCXAacAG4CzkmyY1+0dwONVdQxwKXBJ334GsKqqjgOOB85Nsm7gNiVJy2jMI5KNwJaqeqCqdgDXAJvm9dkEXN1/vg44KUmAAlYnWQkcBOwAnhq4TUnSMhozSI4EHp6zvLVvW7BPVe0EngQOowuVp4FHgYeAD1fV9oHblCQto5UjbjsLtNXAPhuB54AjgFcA/yvJzQO32W04OQc4B+BVr3rVwJIlSS/VmEckW4Gj5iyvBR7ZXZ9+GusQYDvwVuCGqnq2qh4DvgzMDNwmAFV1eVXNVNXMmjVr9sBwJEkLGTNIbgOOTbI+yYHAmcDmeX02A2f3n08HbqmqopvOOjGd1cAbgPsGblOStIxGm9qqqp1JzgNuBFYAV1bVPUk+AMxW1WbgCuDjSbbQHYmc2a9+GXAVcDfddNZVVXUnwELbHGsMkqTFpTsA2LfNzMzU7OzspMuQpKmR5PaqmhnS1zvbJUlNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNDBJJUhODRJLUxCCRJDUxSCRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEktTEIJEkNTFIJElNDBJJUpNU1aRrGF2SbcA3lrj64cC392A5k7SvjGVfGQc4lr3RvjIOaBvLD1TVmiEd94sgaZFktqpmJl3HnrCvjGVfGQc4lr3RvjIOWL6xOLUlSWpikEiSmhgki7t80gXsQfvKWPaVcYBj2RvtK+OAZRqL50gkSU08IpEkNTFIBkjy/iTfTHJH//NTk66pRZJ/k6SSHD7pWpYqyQeT3Nn/fXwuyRGTrmmpknwoyX39eD6T5NBJ17QUSc5Ick+S55NM5VVPSU5Ocn+SLUkumHQ9S5XkyiSPJbl7OfZnkAx3aVW9tv/57KSLWaokRwH/AHho0rU0+lBV/UhVvRa4HnjfpAtqcBPww1X1I8CfAxdOuJ6luht4M/DFSReyFElWAJcBpwAbgLOSbJhsVUv2O8DJy7Uzg2T/cynwHmCqT45V1VNzFlczxeOpqs9V1c5+8U+BtZOsZ6mq6t6qun/SdTTYCGypqgeqagdwDbBpwjUtSVV9Edi+XPszSIY7r596uDLJKyZdzFIkORX4ZlV9bdK17AlJfjXJw8DbmO4jkrl+Hvifky5iP3Uk8PCc5a19mxaxctIF7C2S3Az8nQW+ugj4KPBBut96Pwj8Ot0/+L3OIuP4d8A/XN6Klu7FxlJVf1BVFwEXJbkQOA+4eFkLfAkWG0vf5yJgJ/CJ5aztpRgyjimWBdqm9kh3ORkkvar6ySH9kvwXujn5vdLuxpHkOGA98LUk0E2ffCXJxqr61jKWONjQvxPgvwP/g704SBYbS5KzgZ8BTqq9+Jr8l/B3Mo22AkfNWV4LPDKhWqaKU1sDJPn+OYun0Z1UnCpVdVdVfV9VrauqdXT/aF63t4bIYpIcO2fxVOC+SdXSKsnJwHuBU6vqf0+6nv3YbcCxSdYnORA4E9g84ZqmgjckDpDk48Br6Q5zHwTOrapHJ1pUoyQPAjNVNZVPOU3yaeDVwPN0T3Z+V1V9c7JVLU2SLcAq4K/6pj+tqndNsKQlSXIa8J+ANcATwB1V9Y8mW9VL01/a/5vACuDKqvrVCZe0JEk+Cfx9uqf//iVwcVVdMdr+DBJJUguntiRJTQwSSVITg0SS1MQgkSQ1MUgkSU0MEklSE4NEWkSS7yzTfn4nyemtfaTlZpBIkpoYJNJA6Xwoyd1J7krylr79ZUl+u3+p0/VJPvtiRw1J3pfktn47l6d/+Nm8Pg8muSTJn/U/x8z5+k1J/jjJA7v2k+TgJH+U5Ct9bVP5+HNNJ4NEGu7NdI/K+VHgJ4EP9c9hezOwDjgOeCfw44ts5yNV9fqq+mHgILqHNS7kqaraCHyE7rEdu3w/8BP9er/Wtz0DnFZVrwNOAH59oYCSxmCQSMP9BPDJqnquqv4S+ALw+r79d6vq+f4hmJ9fZDsnJLk1yV3AicAP7abfJ+f8OTecfr/f19eBV/ZtAf5DkjuBm+neo/FKpGXgY+Sl4Xb3G/7g3/yTvBz4bboHZj6c5P3Ay3fTvXbz+f8usO+30T0s8fiqerZ/KOfutivtUR6RSMN9EXhLkhVJ1gBvAv4M+BLwT/pzJa+ke+rq7uz6z/3bSQ4GXuwKrLfM+fNPFqntEOCxPkROAH5gkf7SHuMRiTTcZ+immL5Gd4Twnqr6Vv9I+5Po3lPz58CtwJMLbaCqnuhfjnYX3SsJbnuR/a1KcivdL3xnLVLbJ4A/TDIL3MEUv59F08fHyEt7QJKDq+o7SQ6jO0p5Y8tLw6b9fTHav3hEIu0Z1yc5FDgQ+OC0vnlSWgqPSKSRJPkMsH5e83ur6sZJ1CONxSCRJDXxqi1JUhODRJLUxCCRJDUxSCRJTQwSSVKT/wdSfhlTR8fwBgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.linear_model import RidgeClassifier\n",
    "\n",
    "# Ridge Classifier (tune alpha)\n",
    "log_alpha_list = [-5,-4,-3,-2,-1,0,1]\n",
    "ridge_accuracy_temp = []\n",
    "for log_alpha in log_alpha_list:\n",
    "    ridge_model_temp = RidgeClassifier(alpha=np.exp(log_alpha), fit_intercept=True).fit(X_train, y_train)\n",
    "    ridge_accuracy_temp.append(ridge_model_temp.score(X_test, y_test))\n",
    "    \n",
    "plt.plot(log_alpha_list, ridge_accuracy_temp)\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('log_alpha')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ridge test accuracy:\n",
      " 0.08274036074797286\n",
      "\n",
      "\n",
      "Ridge confusion matrix:\n",
      " [[ 50  16   3 ...   3   0  61]\n",
      " [ 22  14   2 ...   1   0  47]\n",
      " [ 23   8   3 ...   6   0  28]\n",
      " ...\n",
      " [ 15   5   3 ...   8   0  35]\n",
      " [  9   3   0 ...   4   0  48]\n",
      " [ 15   8   2 ...   2   0 245]]\n"
     ]
    }
   ],
   "source": [
    "log_alpha = log_alpha_list[ridge_accuracy_temp.index(max(ridge_accuracy_temp))]\n",
    "ridge_model = RidgeClassifier(alpha=np.exp(log_alpha), fit_intercept=True).fit(X_train, y_train)\n",
    "ridge_predictions = ridge_model.predict(X_test)\n",
    "ridge_accuracy = ridge_model.score(X_test, y_test)\n",
    "ridge_cm = confusion_matrix(y_test, ridge_predictions)\n",
    "print('Ridge test accuracy:\\n', ridge_accuracy)\n",
    "print('\\n')\n",
    "print('Ridge confusion matrix:\\n', ridge_cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RF test accuracy:  0.07496276683766341\n",
      "\n",
      "\n",
      "RF confusion matrix:\n",
      " [[ 65  22   6 ...   4   7  32]\n",
      " [ 37  30   5 ...   2   1  18]\n",
      " [ 17  19  13 ...   3   2  12]\n",
      " ...\n",
      " [ 34  12  11 ...  18   1  13]\n",
      " [ 17  14   6 ...   3   2  20]\n",
      " [ 22  18   7 ...   3   2 193]]\n",
      "\n",
      "\n",
      "RF important features:\n",
      " [0.0150277  0.01738822 0.01792349 0.02604569 0.01787894 0.01507862\n",
      " 0.01075832 0.01012372 0.01313021 0.01977903 0.00700512 0.01290174\n",
      " 0.01140333 0.01259201 0.01857053 0.01248465 0.01103925 0.00806744\n",
      " 0.00475292 0.01373874 0.00475452 0.01219254 0.01762476 0.01366448\n",
      " 0.00977383 0.00949344 0.02064301 0.00831872 0.0144555  0.008513\n",
      " 0.00547744 0.01162499 0.01250208 0.01431245 0.01696738 0.01721237\n",
      " 0.00602541 0.01298023 0.01491649 0.00741958 0.01089488 0.01194042\n",
      " 0.01492996 0.0241604  0.00597269 0.01561731 0.00833931 0.00791486\n",
      " 0.00227837 0.02615444 0.01061154 0.00714194 0.01003941 0.01565031\n",
      " 0.00949518 0.00598444 0.01763665 0.00378179 0.01277911 0.00654371\n",
      " 0.00568192 0.00529629 0.00525011 0.0045723  0.         0.00578903\n",
      " 0.00939977 0.00354187 0.         0.00589632 0.0033641  0.01701938\n",
      " 0.02224257 0.01027593 0.00914801 0.00059559 0.00218202 0.00029243\n",
      " 0.00183478 0.00374271 0.00268869 0.00072825 0.00163735 0.00043359\n",
      " 0.00044284 0.00133197 0.00126386 0.00058821 0.00105452 0.\n",
      " 0.00096477 0.         0.00057455 0.0009922  0.00182541 0.00072917\n",
      " 0.00069852 0.00112626 0.00048339 0.00125547 0.         0.00085345\n",
      " 0.00067798 0.00045242 0.         0.00074971 0.00205228 0.00121709\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.01884756 0.00074615 0.00265089 0.00077121\n",
      " 0.00090212 0.01075479 0.00289798 0.00522247 0.00234967 0.00599109\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.06549041]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Random Forest Classifier (tune n_estimators)\n",
    "# Can only handle small n_estimators, or the memory will exceed.\n",
    "\n",
    "rf_model = RandomForestClassifier(n_estimators = 40, n_jobs = -1).fit(X_train, y_train)\n",
    "rf_predictions = rf_model.predict(X_test)\n",
    "rf_accuracy = rf_model.score(X_test, y_test)\n",
    "rf_cm = confusion_matrix(y_test, rf_predictions)\n",
    "print('RF test accuracy: ', rf_accuracy)\n",
    "print('\\n')\n",
    "print('RF confusion matrix:\\n', rf_cm)\n",
    "print('\\n')\n",
    "print('RF important features:\\n', rf_model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'n_neighbors')"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4VOXZx/HvnX1hD2HfAmELJChExF3BBVDZbdVqbW3Vt2prrZWlbogboK3t+5baatUqWjdARFRwwYVSQAJKEvawJAQChCUhBLLf7x9z0BgDiZCTM5Pcn+uai5kzzznzezjJ3JlzzjyPqCrGGGNMXQvyOoAxxpiGyQqMMcYYV1iBMcYY4worMMYYY1xhBcYYY4wrrMAYY4xxhRUYY4wxrrACY4wxxhVWYIwxxrgixOsAXmrdurV269bN6xjGGBNQVq9evV9VY2tq16gLTLdu3UhJSfE6hjHGBBQRyaxNOztEZowxxhVWYIwxxrjCCowxxhhXuFpgRGS4iGwSkQwRmVzN8+Ei8obz/EoR6eYsDxWRl0QkTUQ2iMiUKusFi8hXIrKw0rI4ZxtbnG2Gudk3Y4wxJ+dagRGRYGAWMAJIAK4TkYQqzX4BHFLVeOBpYIaz/BogXFUTgUHAbceLj+MuYEOVbc0AnlbVnsAhZ9vGGGM84uYnmMFAhqpuU9US4HVgdJU2o4GXnPtzgGEiIoAC0SISAkQCJcBhABHpBFwJ/PP4Rpx1hjrbwNnmGDc6ZYwxpnbcLDAdgZ2VHmc7y6pto6plQD4Qg69QFAI5QBbwlKoedNb5MzARqKi0nRggz9nGiV7LGGNMPXKzwEg1y6rOz3yiNoOBcqADEAfcIyLdReQqYJ+qrj6F1/I1FLlVRFJEJCU3N/ekHTDGmIbmYGEJ//h8K0eKy2pufJrc/KJlNtC50uNOwO4TtMl2Doc1Bw4C1wOLVLUU2Cciy4Bk4ExglIiMBCKAZiLyCnAj0EJEQpxPMdW9FgCq+izwLEBycnK1RcgYYxoSVWVNVh6vrMjkvdQcSsor6BoTxfD+7V19XTcLzCqgp4jEAbuAa/EVjsoWADcBy4EJwBJVVRHJAoY6xSMKGAL8WVXfBKYAiMjFwO9V9Qbn8afONl53tvmOi30zxhi/d7SkjHe+3s3s5ZmszzlM0/AQrj+7Cz85uws92zZ1/fVdKzCqWiYidwKLgWDgBVVdJyLTgBRVXQA8D8wWkQx8n1yudVafBbwIpOM7/PWiqqbW8JKTgNdF5FHgK2fbxhjT6GTsK+CVFVnMXZ1NQXEZfds34/GxiYw+owPR4fU3QpioNt6jRMnJyWpjkRljGoLS8go+XLeXV1ZksnzbAcKCg7gyqT03DOnKwC4t8F1sWzdEZLWqJtfUrlEPdmmMMYEuJ/8Yr325k9e/zGJfQTGdWkYyaXgffpTciZgm4Z5mswJjjDEBRlVZlnGAV1Zk8tGGvVSocknvNtw4pCsX9oolOKjuPq2cDiswxhgTIPKPljJnTTavrshk2/5CWkWHccsF3fnJ2V3o3CrK63jfYwXGGGP8XFp2Pq+syOSdtbsoKq1gUNeW/HlYT0YktiM8JNjreCdkBcYYY/xQUWk5C1NzmL0ik7U784gMDWbsmZ24YUgX+nVo7nW8WrECY4wxfiTzQCGvrszizZSd5B0tJb5NE6ZencC4QZ1oFhHqdbwfxAqMMcZ4rLxCWbJxH7NXZPLF5lxCgoQr+rXjhiFdGdK9VZ1eYlyfrMAYY4xHcguKeTNlJ/9emcWuvGO0axbB3Zf24trBnWnbLMLreKfNCowxxtQjVWXVjkPMXpHJovQcSsuV8+Nb88BVCVzatw0hwQ1nomErMMYYUw8KikqZ/9UuXlmRxaa9BTSLCOHGId34yZAu9Iht4nU8V1iBMcYYF23cc5hXVmTy9ppdFJaU079jM2aOT+LqAR2IDPPfS4zrghUYY4ypYyVlFXyQnsOrK7L4csdBwkKCuDqpAzee05UBnZoH7En7H8oKjDHG1JFdecf498pM3li1k/1HSugaE8V9I/syYVAnWkaHeR2v3lmBMcaY01BRoSzN2M/s5Zks2bgXgKF92nLjOV25IL41QX4yLpgXrMAYY8wpOFRYwlurd/LqyiwyDxyldZMwbr84nuvO7kLHFpFex/MLVmCMMaaWVJW12fnMXp7Ju6m7KSmrYHC3VtxzeW+G92tHWEjDucS4LliBMcaYGhwrKWfB2l3MXpFJ+q7DRIcF8+PkztwwpCu927k/9XCgsgJjjDEnsDX3CK+uyGLO6p0cLiqjd9umPDKmP2PP7EiTepx6OFDZ/5AxxlRSVl7Bxxv2MntFJssyDhAaLIzo354bz+lKcteWjeYS47pgBcYYY4C9h4t4/cudvPZlFnsOF9GxRST3XtGbHyV3Jrapt1MPByorMMaYRktVWb7NN/Xwh+v2UlahXNQrlkfH9OeSPm38ZurhQGUFxhjT6BwuKmXe6mxmr8hka24hLaJCufn8OK4f3IVuraO9jtdgWIExxjQa63b7ph6e/9VujpWWc0bnFjx1zQCuSmpPRGjDHhfMC1ZgjDENWlFpOR+k5zB7eSZrsvKICA1i9ICO3DCkK4mdAmPq4UBlBcYY0yDtPHiUV1Zm8uaqnRw6Wkr31tE8cFUCEwZ2onlUYE09HKiswBhjGozyCuXzzfuYvTyTzzbnEiTCZX1944Kd2yPGLjGuZ1ZgjDEB78CRYt5I2cmrK3xTD7dpGs6vh/bkusGdad/cxgXziqsFRkSGA38BgoF/qur0Ks+HAy8Dg4ADwI9VdYeIhAL/BAY6GV9W1SdEJAL4Agh3ls9R1Yecbf0LuAjIdzb/M1X92s3+GWO8o6qsyTrE7OWZvJ+2h5LyCs7pHsN9V/blsoS2hDagqYcDlWsFRkSCgVnAZUA2sEpEFqjq+krNfgEcUtV4EbkWmAH8GLgGCFfVRBGJAtaLyGtAJjBUVY84Reg/IvKBqq5wtnevqs5xq0/GGO8VFpcx/+tdzF6eycY9BTQND+H6s7tww5AuxLexccH8iZufYAYDGaq6DUBEXgdGA5ULzGhgqnN/DvBX8R0kVSBaREKASKAEOKyqChxx2oc6N3WxD8YYP7FlbwGvrMhk7ppdHCkuI6F9M54Yl8ioAR2ItnHB/JKbe6UjsLPS42zg7BO1UdUyEckHYvAVm9FADhAF3K2qB+GbT0argXhglqqurLS9x0TkQeATYLKqFtd5r4wx9aakrIIP1+9h9vJMVm4/SFhwEFcmteeGIV0Z2KWFnbT3c24WmOr2fNVPGydqMxgoBzoALYGlIvKxqm5T1XLgDBFpAbwtIv1VNR2YAuwBwoBngUnAtO+FErkVuBWgS5cup9QxY4y7cvKP8drKLF5btZPcgmI6tYxk8og+XDOoEzFNbFywQOFmgckGOld63AnYfYI22c7hsObAQeB6YJGqlgL7RGQZkAxsO76iquaJyGfAcCBdVXOcp4pF5EXg99WFUtVn8RUgkpOT7fCaMX6iokJZttU39fDHG/aiwCW923DjkK5c2CvWxgULQG4WmFVATxGJA3YB1+IrHJUtAG4ClgMTgCWqqiKSBQwVkVfwHSIbAvxZRGKBUqe4RAKX4rswABFpr6o5zjmcMUC6i30zxtSR/KOl30w9vH1/Ia2iw7jtoh5cP7gLnVtFeR3PnAbXCoxzTuVOYDG+y5RfUNV1IjINSFHVBcDzwGwRycD3yeVaZ/VZwIv4ioQAL6pqqogkAS8552GCgDdVdaGzzqtOARLga+B/3OqbMeb0pWbnfTP1cFFpBYO6tuSuYT0ZkdiO8BAbF6whEN+FWY1TcnKypqSkeB3DmEajqLScd9fu5pUVmazNzicqLJgxZ3bkhrO7ktChmdfxTC2JyGpVTa6pnV3bZ4xx3fb9hby6IpO3VmeTf6yU+DZNeHhUP8YO7EizCBsXrKGyAnMK1u7MY9WOg/zygu5eRzHGr6Vm5/Hk4k0s3bKfkCDhiv7tuHFIV86Oa2WXGDcCVmBOwdw1vomKzuzSgkFdW3kdxxi/lHe0hJv/5TsE/bvLenHtWZ1p0yzC41SmPtlgPadg4vA+dGgeycQ5qRSVlnsdxxi/9Oh7G8g7WsJLN5/Fb4b1tOLSCFmBOQVNwkN4bGx/tuYW8tclGV7HMcbvfL45lzmrs/mfi3rQr4NN6tVYWYE5RRf3bsO4gR155vOtrNudX/MKxjQSR4rL+MO8NHrERvPrYfFexzEesgJzGh68KoGWUWFMnJNKaXmF13GM8QszF21kd/4xZk4YYN9naeSswJyGFlFhPDK6H+t2H+a5pdtqXsGYBu7L7Qd5eXkmPzu3G4O6tvQ6jvGYFZjTNCKxPSP6t+PPH29ha+6RmlcwpoEqKi1n0txUOreK5N4rensdx/gBKzB14OHR/YgMDWbSnFQqKhrvyAimcfvzx1vYvr+Q6eOSiAqzb0AYKzB1ok3TCB64KoGUzEO8vHyH13GMqXdp2fk8t3QbP07uzHnxrb2OY/yEFZg6Mn5gRy7qFcvMxZvYefCo13GMqTclZRXcO2ctrZuE8Ycr+3odx/gRKzB1RER4fFwiAvzh7TQa8yCipnH5x+db2bingEfHJNI80sYVM9+yAlOHOrbwzbq3dMt+3lqd7XUcY1y3ZW8B/7ckg6sHdOCyhLZexzF+xgpMHfvJ2V0Z3K0Vjy5cz77DRV7HMcY15RXKvXNSiQ4PZurVCV7HMX7ICkwdCwoSpo9PpLisgvvnp9uhMtNgvbhsO1/vzGPqqH7ENAn3Oo7xQ1ZgXNA9tgl3X9aLD9fv5f20PV7HMabOZR04ylMfbmJYnzaMGtDB6zjGT1mBcckvz48jsWNzHlqQzqHCEq/jGFNnVJXJ81IJDQri0bH9bV4Xc0JWYFwSEhzEjPFJ5B0tZdrC9V7HMabOvL5qJ//deoApI/vSvnmk13GMH7MC46KEDs24/eIevP3VLj7duM/rOMactpz8Yzz+3gbO6R7DdYM7ex3H+DkrMC67Y2g8Pds04Q9vp1FQVOp1HGNOmapy/9vplFZUMH18oh0aMzWyAuOy8JBgZk5IYu/hIqZ/sNHrOMacsgVrd/PJxn38/vLedI2J9jqOCQBWYOrBmV1acvN5cby6MovlWw94HceYH+zAkWIefnc9Z3Ruwc/Pi/M6jgkQVmDqyT2X96ZLqygmz0vlWEm513GM+UGmvruegqJSZk5IIjjIDo2Z2rECU08iw4KZPj6RzANHefrjzV7HMabWPlq/l3fX7ubXQ3vSq21Tr+OYAGIFph6d26M11w3uwj+XbmPtzjyv4xhTo/xjpdw/P40+7Zryq4t7eB3HBBgrMPVsysg+tGkawcQ5qZSUVXgdx5iTeuL9DeQWFPPkhAGEBtvbhflh7CemnjWLCOWxsf3ZtLeAWZ9meB3HmBNalrGf11ft5JYLu5PYqbnXcUwAcrXAiMhwEdkkIhkiMrma58NF5A3n+ZUi0s1ZHioiL4lImohsEJEpzvIIEflSRNaKyDoRebjStuKcbWxxthnmZt9Ox7C+bRl9Rgf+9lkGG/cc9jqOMd9ztKSMyfNSiWsdzd2X9vI6jglQrhUYEQkGZgEjgATgOhGpOqb3L4BDqhoPPA3McJZfA4SraiIwCLjNKT7FwFBVHQCcAQwXkSHOOjOAp1W1J3DI2bbfeujqfjSLCGXSnFTKyu1QmfEvTy3ezM6Dx5gxPomI0GCv45gA5eYnmMFAhqpuU9US4HVgdJU2o4GXnPtzgGHi+3qwAtEiEgJEAiXAYfU54rQPdW7qrDPU2QbONse41K860So6jKmj+rE2O58Xlm33Oo4x31ideYgX/7udn57TlcFxrbyOYwKYmwWmI7Cz0uNsZ1m1bVS1DMgHYvAVikIgB8gCnlLVg+D7ZCQiXwP7gI9UdaWzTp6zjRO9Fs76t4pIioik5Obmnn4vT8NVSe25tG9b/vjhZrbvL/Q0izEAxWXlTJqbSofmkUwc3sfrOCbAuVlgqvs2VtXZt07UZjBQDnQA4oB7RKQ7gKqWq+oZQCdgsIj0r+Vr4az/rKomq2pybGxs7XriEhHhsbH9CQsJYvLcVCoqbHIy463/+ySDjH1HeHxcIk3CQ7yOYwKcmwUmG6g83GonYPeJ2jiHw5oDB4HrgUWqWqqq+4BlQHLlFVU1D/gMGA7sB1o42zjRa/mlts0iuP/KvqzcfpB/f5nldRzTiK3bnc8zn29l/MBOXNTL2z++TMPgZoFZBfR0ru4KA64FFlRpswC4ybk/AViivjmGs4Ch4hMNDAE2ikisiLQAEJFI4FJgo7POp842cLb5jot9q1M/Su7M+fGtmf7BRnbnHfM6jmmEysormDgnlZZRYTxwVV+v45gGwrUC45wPuRNYDGwA3lTVdSIyTURGOc2eB2JEJAP4HXD8UuZZQBMgHV+helFVU4H2wKcikuos/0hVFzrrTAJ+52wrxtl2QBARnhiXSHmF8oe30/DVS2Pqz7NLt7Fu92EeHdOPFlF+e4W/CTDSmN/MkpOTNSUlxesY33jhP9uZtnA9f/rRAMYN7OR1HNNIbM09woi/LGVYnzY8c8Mgr+OYACAiq1U1uaZ29k1+P3LTud0Y2KUF0xauJ7eg2Os4phGoqFAmzUklMjSYh0f38zqOaWCswPiR4CBh5oQkjhaXM3XBOq/jmEZg9opMUjIP8eBVCbRpGuF1HNPA1KrAiMhcEblSRKwguSy+TVPuurQn76XlsCh9j9dxTAO28+BRZizayEW9Yhk3sNqvjRlzWmpbMJ7Bd+nwFhGZLiL2DSwX3XphdxLaN+OBd9LJP1rqdRzTAKn6LigR4PFxifgGwzCmbtWqwKjqx6r6E2AgsAP4SET+KyI/F5FQNwM2RqHBQcyckMTBwhIefW+913FMA/TW6myWbtnP5BF96Ngi0us4poGq9SEvEYkBfgb8EvgK+Au+gvORK8kauf4dm3Pbhd15a3U2X2z2dkgb07DsO1zEowvXMziuFT85u6vXcUwDVttzMPOApUAUcLWqjlLVN1T11/i+r2Jc8JthPekRG82UeWkUFpfVvIIxNVBVHngnneKyCqaPSyQoyA6NGffU9hPMX1U1QVWfUNWcyk/U5lpoc2oiQoOZMT6J3fnHmLloo9dxTAPwftoeFq/by92X9aJ7rP1taNxV2wLT9/gQLQAi0lJEbncpk6kkuVsrbjqnGy+vyGTVjoNexzEB7FBhCQ8tSCexY3N+eX6c13FMI1DbAnOLM7gkAKp6CLjFnUimqnuv6E3HFpFMmptKUWm513FMgJq2cD15R0uZOSGJkGD7xoFxX21/yoKk0nWMzmyVNmBRPYkOD+GJcYlsyy3kL59s8TqOCUCfbtzH21/t4vZL4unbvpnXcUwjUdsCsxh4U0SGichQ4DVgkXuxTFUX9IzlR8mdePaLbaTvyvc6jgkgBUWl/OHtNHq2acIdl/TwOo5pRGpbYCYBS4BfAXcAnwAT3QplqnffyARaRYdx75xUSssrvI5jAsT0Dzay93ARMyckER4S7HUc04jU9ouWFar6jKpOUNXxqvoPVbWTAfWseVQoj47pz4acw/zj861exzEBYPnWA7y6Moubz4vjzC4tvY5jGpnafg+mp4jMEZH1IrLt+M3tcOb7rujXjiuT2vO/n2SwZW+B13GMHztWUs6Ueal0jYninst7ex3HNEK1PUT2Ir7xyMqAS4CXgdluhTIn9/CofkSHBzNxbirlFY13Ph9zck9/vJkdB47yxLhEIsPs0Jipf7UtMJGq+gm+CcoyVXUqMNS9WOZkWjcJ56Gr+/FVVh7/+u8Or+MYP7R2Zx7/XLqN6wZ34dwerb2OYxqp2haYImeo/i0icqeIjAXauJjL1GD0GR24pHcsTy3eRNaBo17HMX6kpKyCiXNSadM0gikjbeBz453aFpjf4huH7DfAIOAG4Ca3QpmaiQiPjU0kOEiY8nYqjXnqa/Ndf/ssg017C3h8XH+aRdhg58Y7NRYY50uVP1LVI6qarao/d64kW1EP+cxJdGgRyZSRfViWcYA3Vu30Oo7xA5v2FDDr0wzGnNGBoX3aeh3HNHI1FhjncuRBYjMS+aXrzurCkO6teOy9DezJL/I6jvFQWXkFE+espVlEKA9e3c/rOMbU+hDZV8A7InKjiIw7fnMzmKmdoCBh+rgkSisquH9+mh0qa8ReWLadtdn5TB3Vj1bRNpKT8V5tC0wr4AC+K8eudm5XuRXK/DDdWkdzz2W9+XjDPt5Nzal5BdPgbN9fyB8/3MxlCW25Kqm913GMASCkNo1U9eduBzGn5+bz41iYlsPUBes4r0cMMU3CvY5k6klFhTJ5biphIUE8OqY/djTb+IvafpP/RRF5oerN7XCm9oKDhJnjkygoKuXhd9d7HcfUo39/mcXK7Qe5/8q+tG0W4XUcY75R20NkC4H3nNsnQDPgiFuhzKnp3a4pd1wSz4K1u/l4/V6v45h6sDvvGNM/2Mj58a35UXJnr+MY8x21PUQ2t/JjEXkN+NiVROa03H5xPIvS93D//HQGd29l34NowFSVP7ydRnmF8sS4RDs0ZvzOqU5r1xPoUlMjERkuIptEJENEJlfzfLiIvOE8v1JEujnLQ0XkJRFJE5ENIjLFWd5ZRD51lq0TkbsqbWuqiOwSka+d28hT7FtACwsJYsb4JPYVFPHE+xu8jmNcNP/rXXy2KZeJw3vTuVWU13GM+Z7anoMpEJHDx2/Au/jmiDnZOsHALGAEkABcJyIJVZr9AjikqvHA08AMZ/k1QLiqJuIbOeA2p/iUAfeoal9gCHBHlW0+rapnOLf3a9O3hmhA5xbcckF3XvtyJ//N2O91HOOC3IJiHn53PYO6tuSn53TzOo4x1artfDBNVbVZpVuvqofNqjEYyFDVbapaArwOjK7SZjTwknN/DjDM+UKnAtEiEgJEAiXAYVXNUdU1TqYCYAPQsTZ9aGzuvqwX3WKimDwvjaMlZV7HMXVs6oJ1HC0uZ8Z433BBxvij2n6CGSsizSs9biEiY2pYrSNQefySbL5fDL5po6plQD4Qg6/YFAI5QBbwlKoerJKpG3AmsLLS4jtFJNW5yq1Rz64UERrMjPFJZB08yh8/3Ox1HFOHFqXv4b20HO66tCfxbZp6HceYE6rtOZiHVPWbieBVNQ94qIZ1qvuzqurXzE/UZjBQDnQA4oB7RKT7NyuJNAHmAr9V1cPO4meAHsAZ+ArTH6sNJXKriKSISEpubm4NXQhsZ3eP4YYhXXhh2XbWZB3yOo6pA/lHS3ngnXQS2jfj1gu717yCMR6qbYGprl1NV6BlA5Wvm+wE7D5RG+dwWHPgIHA9sEhVS1V1H7AMSHbaheIrLq+q6rzjG1LVvaparqoVwHP4itT3qOqzqpqsqsmxsbE1dCHwTRreh/bNIpg4J5XiMpvlOtA9+t56DhaWMHNCEqHBp3qNjjH1o7Y/oSki8icR6SEi3UXkaWB1DeusAnqKSJyIhAHXAguqtFnAt8P+TwCWqG8wrSxgqPhE4zuhv9E5P/M8sEFV/1R5QyJSeXyMsUB6LfvWoDWNCOWxcYlk7DvCrCUZXscxp+GLzbm8tTqb2y7sTv+OzWtewRiP1bbA/BrfifY3gDeBY8AdJ1vBOadyJ7AY38n4N1V1nYhME5FRTrPngRgRyQB+Bxy/lHkW0ARfkVgFvKiqqcB5wI34ik/Vy5FnOpc1p+Kb1vnuWvatwbukdxvGndmRv322lfW7D9e8gvE7hcVlTJmXRo/YaH4zrKfXcYypFWnMo+8mJydrSkqK1zHqxaHCEi57+nPaN4/k7dvPJcQOrwSUh95J5+UVmcz5n3MY1LWV13FMIyciq1U1uaZ2tb2K7CMRaVHpcUsRWXw6AU39ahkdxrTR/Unblc9zS7d7Hcf8AKt2HOTlFZncdE43Ky4moNT2z9jWzpVjAKjqIaCNO5GMW0Ymtmd4v3Y8/fFmtubaUHKBoKi0nElzU+nYIpJ7r+jtdRxjfpDaFpgKEflmaBjnOyiN99haAJs2ph8RIUFMnptKRYXtQn/3l0+2sC23kCfGJRIdXquhA43xG7UtMPcB/xGR2SIyG/gcmOJeLOOWNk0jeOCqBFbtOMQrKzO9jmNOIn1XPs9+sY0fJXfigp4N/5J60/DUdqiYRfi+h7IJ35Vk9+C7kswEoAmDOnFhr1hmfLCR7ENHvY5jqlFaXsG9c1KJiQ7jviurDuFnTGCo7Un+X+KbB+Ye5zYbmOpeLOMmEeHxsf0BmDIvjcZ8JaG/+sfnW9mQc5hHxvSneaRNuWACU20Pkd0FnAVkquol+MYAa9jjrDRwnVpGMWlEH5Zu2c/cNbu8jmMq2bK3gP/9JIMrk9pzRb92Xscx5pTVtsAUqWoR+OZwUdWNgF3SEuBuOLsrZ3VrySML17OvoMjrOAYor1Amzk0lOjyYh0f18zqOMaeltgUm2/kezHzgIxF5h++PK2YCTFCQMH18EsdKy3lw/jqv4xjgpf/u4KusPB66uh+tm4R7HceY01Lbk/xjVTVPVacCD+Ab4qWm4fpNAOgR24S7L+3FonV7eD8tx+s4jVrWgaM8uXgTQ/u0YfQZHbyOY8xp+8Hjhajq56q6wJlEzDQAt1wQR2LH5jz4Tjp5R223ekFVmfJ2KsFBwqNj+uMb19WYwGYDUhlCgoOYMT6JvKOlTFu43us4jdIbq3ayLOMAU0b2oUOLSK/jGFMnrMAYABI6NONXF/dg3ppdfLppn9dxGpU9+UU89t4GhnRvxXVndal5BWMChBUY8407h8bTs00T7puXRkFRqddxGgVV5f75aZRWVDB9XBJBQXZozDQcVmDMN8JDgpkxIYmcw0XMXLTJ6ziNwrupOXy8YR/3XNabbq2jvY5jTJ2yAmO+Y2CXltx8XhyzV2SyctsBr+M0aAeOFDN1wToGdG7BzefHeR3HmDpnBcZ8zz2X96JLqygmz0ujqLTc6zgN1sPvrqegqJQnJyQRbIfGTANkBcZ8T1RYCNPHJbJ9fyFPf7TZ6zgN0sfr97Jg7W7uvKQnvdo29TqOMa6wAmOqdW58a64b3Jnnlm4jNTuv5hVMrR0uKuX++en0adeUX13cw+s4xrjGCoxf1D5FAAAWD0lEQVQ5oSkj+xLbNJyJc1IpKavwOk6D8cT7G9hXUMSM8UmEhdivoGm47KfbnFCziFAeG5PIxj0FPPPZVq/jNAj/zdjPa1/u5JYLujOgcwuv4xjjKisw5qQuTWjLqAEd+OunW9i0p8DrOAHtaEkZk+elEdc6mrsv6+V1HGNcZwXG1OihqxNoGhHKxLmplFfY5GSn6o8fbibr4FGmj0skIjTY6zjGuM4KjKlRTJNwpo7qx9qdeby4bLvXcQLSmqxDvLBsOzcM6cLZ3WO8jmNMvbACY2rl6qT2XNq3LU99uIkd+wu9jhNQisvKmTgnlfbNIpg0vI/XcYypN1ZgTK2I+IaRDw0KYvK8VCrsUFmt/XVJBhn7jvD4uESaRoR6HceYemMFxtRau+YR3HdlX1ZsO8hrq7K8jhMQ1u8+zDOfbWXcwI5c3LuN13GMqVeuFhgRGS4im0QkQ0QmV/N8uIi84Ty/UkS6OctDReQlEUkTkQ0iMsVZ3llEPnWWrRORuyptq5WIfCQiW5x/W7rZt8bqx2d15rz4GJ54fyM5+ce8juPXysormDQ3lRZRoTxwZYLXcYypd64VGBEJBmYBI4AE4DoRqfpb9gvgkKrGA08DM5zl1wDhqpoIDAJuc4pPGXCPqvYFhgB3VNrmZOATVe0JfOI8NnVMRHhibBLlFcp9b6ejaofKTuS5pdtJ25XPtNH9aRkd5nUcY+qdm59gBgMZqrrNmV75dWB0lTajgZec+3OAYeKbK1aBaBEJASKBEuCwquao6hoAVS0ANgAdq9nWS8AYd7plusRE8fsrerNk4z7e+Xq313H80tbcIzz98WaG92vHyMT2XscxxhNuFpiOwM5Kj7P5thh8r42qlgH5QAy+YlMI5ABZwFOqerDyis4nmjOBlc6itqqa42wrB7AD3i762bndGNilBQ+/u479R4q9juNXKiqUyXNTiQgJYtqYfl7HMcYzbhaY6sYfr3o85URtBgPlQAcgDrhHRLp/s5JIE2Au8FtVPfyDQoncKiIpIpKSm5v7Q1Y1lQQHCTMnJFFYXM7UBeu8juNXXlmZyaodh3jgqgTaNI3wOo4xnnGzwGQDnSs97gRUPZ7yTRvncFhz4CBwPbBIVUtVdR+wDEh22oXiKy6vquq8StvaKyLtnTbtgWonllfVZ1U1WVWTY2NjT7OLjVt8m6b8Zlg8C1Nz+HDdHq/j+IXsQ0eZ8cFGLuwVy4RBnbyOY4yn3Cwwq4CeIhInImHAtcCCKm0WADc59ycAS9R31jgLGCo+0fhO6G90zs88D2xQ1T+dZFs3Ae/UeY/M99x2UQ/6tm/G/fPTyT9W6nUcT6kqU+alAfD42P74flyNabxcKzDOOZU7gcX4Tsa/qarrRGSaiIxymj0PxIhIBvA7vr3yaxbQBEjHV6heVNVU4DzgRnzF52vnNtJZZzpwmYhsAS5zHhuXhQYH8eSEJA4UlvDYe+u9juOpuWt2sXTLfiaN6EOnllFexzHGc9KYLzNNTk7WlJQUr2M0CDMWbeSZz7byyi/O5vyerb2OU+/2FRRx2Z++oFfbJrxx6zkE2RTIpgETkdWqmlxTO/smv6kTdw3rSffYaCbPS6WwuMzrOPXuwfnrOFZazvTxSVZcjHFYgTF1IiI0mJnjk9iVd4wnF2/yOk69ej8th0Xr9nD3pb3oEdvE6zjG+A0rMKbOJHdrxU+HdOWl5TtI2XGwxvYNQd7REh58J53Ejs255YI4r+MY41eswJg6NXF4Hzo0j2Ti3FSKSsu9juO6aQvXk3e0lBnjkwgJtl8nYyqz3whTp6LDQ3hiXCLbcgv5vyVbvI7jqk837WPeml386uIeJHRo5nUcY/yOFRhT5y7sFcs1gzrx98+3kb4r3+s4rigoKuW+eWn0bNOEO4fGex3HGL9kBca44v4rE2gVHcbEOamUlld4HafOzVi0kZzDRcyYkER4SLDXcYzxS1ZgjCuaR4XyyOj+rM85zLNfbPM6Tp1aue0Ar6zI4ubz4hjYxaYdMuZErMAY1wzv344rE9vzl4+3kLHviNdx6kRRaTmT56XRpVUU91zey+s4xvg1KzDGVVNH9SMqPJhJc1Mprwj8USOe/mgz2/cXMn1cIlFhIV7HMcavWYExroptGs5DVyewOvMQLy/f4XWc07J2Zx7PLd3GdYM7c2584xsOx5gfygqMcd2YMzpyce9YZi7axM6DR72Oc0pKyiqYNDeV2KbhTBnZ1+s4xgQEKzDGdSLC42MTCQ4SpsxLIxAHWH3ms61s3FPAY2MSaRYR6nUcYwKCFRhTLzq0iGTyiD78J2M/b6Vkex3nB9m0p4C/frqFUQM6cGlCW6/jGBMwrMCYenP94C6cHdeKR95bz97DRV7HqZXyCmXi3FSaRoTy0NUJXscxJqBYgTH1JihImD4+iZKyCu6fnx4Qh8peXLadtTvzmDqqHzFNwr2OY0xAsQJj6lVc62juubwXH63fy3tpOV7HOakd+wt56sNNXNq3LVcntfc6jjEBxwqMqXc3nxfHgE7NeeiddRwsLPE6TrUqKpTJ81IJDQri0TH9EbFJxIz5oazAmHoXEhzEzAkDOFxUyrR313kdp1qvrcpixbaD3HdlX9o1j/A6jjEByQqM8UTvdk2545J45n+9myUb93od5zt25x3jifc3cl58DD8+q7PXcYwJWFZgjGduvzie3m2b8od56RQUlXodBwBV5f756ZRXKE+MTbJDY8acBiswxjNhIUHMnJDEvoIinvhgo9dxAHjn690s2biP31/Rmy4xUV7HMSagWYExnhrQuQW/vKA7/16ZxfKtBzzNsv9IMQ+/u46BXVrws3O7eZrFmIbACozx3N2X9qJbTBST56VyrKTcsxwPLVhHYXE5MyckERxkh8aMOV1WYIznIsOCmT4+icwDR/njh5s8ybB43R7eS83hN8PiiW/T1JMMxjQ0VmCMXxjSPYafnN2FF5Zt56usQ/X62vnHSnlgfjp92zfjtot61OtrG9OQWYExfmPyiD60axbBpLmpFJfV36Gyx95bz4HCEp6ckERosP1KGFNXXP1tEpHhIrJJRDJEZHI1z4eLyBvO8ytFpJuzPFREXhKRNBHZICJTKq3zgojsE5H0KtuaKiK7RORr5zbSzb6Zutc0IpTHxiWyee8RZn26tV5ec+mWXN5MyebWC7vTv2PzenlNYxoL1wqMiAQDs4ARQAJwnYhUHY72F8AhVY0HngZmOMuvAcJVNREYBNx2vPgA/wKGn+Bln1bVM5zb+3XVF1N/LundhnFnduRvn2awIeewq69VWFzGlHlpdG8dzV3Derr6WsY0Rm5+ghkMZKjqNlUtAV4HRldpMxp4ybk/Bxgmvm+2KRAtIiFAJFACHAZQ1S+Agy7mNh574KoEWkSFMmluKmXlFa69zpOLN7Er7xgzJiQRERrs2usY01i5WWA6AjsrPc52llXbRlXLgHwgBl+xKQRygCzgKVWtTVG5U0RSncNoLU8zv/FIy+gwHh7Vn9TsfJ7/z3ZXXiNlx0FeWr6Dnw7pylndWrnyGsY0dm4WmOq+SFB1ApATtRkMlAMdgDjgHhHpXsPrPQP0AM7AV5j+WG0okVtFJEVEUnJzc2vYpPHKyMR2XNGvLX/6aDPbco/U6baLSsuZODeVDs0jmTi8T51u2xjzLTcLTDZQeaTATsDuE7VxDoc1x3f463pgkaqWquo+YBmQfLIXU9W9qlquqhXAc/iKVHXtnlXVZFVNjo2NPYVumfogIjwyuj/hIUFMnptGRUXdTU72v59sYVtuIU+MSyQ6PKTOtmuM+S43C8wqoKeIxIlIGHAtsKBKmwXATc79CcAS9U1zmAUMFZ9oYAhw0sGqRKTyjFBjgfQTtTWBoU2zCO6/KoEvdxzk1S+z6mSb6bvy+ccX27hmUCcu7GV/YBjjJtcKjHNO5U5gMbABeFNV14nINBEZ5TR7HogRkQzgd8DxS5lnAU3wFYlVwIuqmgogIq8By4HeIpItIr9w1pnpXNacClwC3O1W30z9uWZQJy7o2Zrp729gV96x09pWaXkFE+ek0io6jPuvrHpBozGmrkkgzIvuluTkZE1JSfE6hqnBzoNHueLPX3BWt1b86+dnnfIQ+rM+zeDJxZv4+w2DGN6/XR2nNKbxEJHVqnrS0xZg3+Q3AaBzqygmDe/D55tzmbdm1yltI2NfAX/5eAtXJra34mJMPbECYwLCjUO6kty1JdMWrie3oPgHrVteoUyck0pUeDBTR/VzKaExpiorMCYgBAUJMyYkcay0nIcW/LDrN15evoM1WXk8eFUCsU3D3QlojPkeKzAmYPSIbcJvL+3J+2l7WJSeU6t1dh48ysxFm7i4dyxjz6z6PV9jjJuswJiAcusF3enfsRn3z19H3tGSk7ZVVabMSyM4SHh8bOIpXxxgjDk1VmBMQAkJDmLm+AHkHS3hkYUbTtr2rZRs/pOxn8kj+tChRWQ9JTTGHGcFxgSchA7N+J+LejB3TTafb65+uJ+9h4t45L31DI5rxfWDu9RzQmMMWIExAerXw+KJb9OEP8xL40hx2XeeU1Xun59OSVkFM8YnERRkh8aM8YIVGBOQwkOCmTE+id35x5i56LujCC1MzeGj9Xu55/JexLWO9iihMcYKjAlYg7q25OfnxvHy8ky+3O6bzeFgYQlTF6xjQKfm3HxenMcJjWncrMCYgPb7K3rRuVUkk+amUlRazrR313G4qJQZE5IICbYfb2O8ZL+BJqBFhYUwfVwS2/cXcvO/VjH/693cfnE8fdo18zqaMY2eFRgT8M6Lb821Z3Xmv1sP0LttU+64JN7rSMYYwGZbMg3ClJF9Ka9Qbj4/jrAQ+7vJGH9gBcY0CM0jQ3nymgFexzDGVGJ/6hljjHGFFRhjjDGusAJjjDHGFVZgjDHGuMIKjDHGGFdYgTHGGOMKKzDGGGNcYQXGGGOMK0RVvc7gGRHJBTJPYdXWwP46juMV64v/aSj9AOuLvzrdvnRV1diaGjXqAnOqRCRFVZO9zlEXrC/+p6H0A6wv/qq++mKHyIwxxrjCCowxxhhXWIE5Nc96HaAOWV/8T0PpB1hf/FW99MXOwRhjjHGFfYIxxhjjCiswVYhIZxH5VEQ2iMg6EbnLWd5KRD4SkS3Ovy2d5SIi/ysiGSKSKiIDve3Bt07Sl6kisktEvnZuIyutM8XpyyYRucK79N8lIhEi8qWIrHX68rCzPE5EVjr75Q0RCXOWhzuPM5znu3mZv7KT9OVfIrK90n45w1nutz9jACISLCJfichC53HA7ZPjqulLoO6THSKS5mROcZbV/3uYqtqt0g1oDwx07jcFNgMJwExgsrN8MjDDuT8S+AAQYAiw0us+1KIvU4HfV9M+AVgLhANxwFYg2Ot+ONkEaOLcDwVWOv/fbwLXOsv/DvzKuX878Hfn/rXAG173oRZ9+RcwoZr2fvsz5uT7HfBvYKHzOOD2yUn6Eqj7ZAfQusqyen8Ps08wVahqjqquce4XABuAjsBo4CWn2UvAGOf+aOBl9VkBtBCR9vUcu1on6cuJjAZeV9ViVd0OZACD3U9aM+f/94jzMNS5KTAUmOMsr7pfju+vOcAwEZF6intSJ+nLifjtz5iIdAKuBP7pPBYCcJ/A9/tSA7/dJydR7+9hVmBOwvkIfya+vzDbqmoO+N64gTZOs47AzkqrZXPyN3FPVOkLwJ3Ox+EXjn9Uxs/74hy++BrYB3yE7xNWnqqWOU0q5/2mL87z+UBM/SY+sap9UdXj++UxZ788LSLhzjJ/3i9/BiYCFc7jGAJ0n/D9vhwXaPsEfH+wfCgiq0XkVmdZvb+HWYE5ARFpAswFfquqh0/WtJplfnVpXjV9eQboAZwB5AB/PN60mtX9pi+qWq6qZwCd8H2y6ltdM+ffgOqLiPQHpgB9gLOAVsAkp7lf9kVErgL2qerqyouraer3++QEfYEA2yeVnKeqA4ERwB0icuFJ2rrWFysw1RCRUHxvyK+q6jxn8d7jHxudf/c5y7OBzpVW7wTsrq+sNamuL6q613mDqwCe49vDYH7dl+NUNQ/4DN/x4hYiEuI8VTnvN31xnm8OHKzfpDWr1JfhziFNVdVi4EX8f7+cB4wSkR3A6/gOjf2ZwNwn3+uLiLwSgPsEAFXd7fy7D3gbX+56fw+zAlOFc0z4eWCDqv6p0lMLgJuc+zcB71Ra/lPnSowhQP7xj6FeO1FfqhxfHQukO/cXANc6V/vEAT2BL+sr78mISKyItHDuRwKX4jun9CkwwWlWdb8c318TgCXqnNH02gn6srHSL7/gOz5eeb/43c+Yqk5R1U6q2g3fSfslqvoTAnCfnKAvNwTaPgEQkWgRaXr8PnA5vtz1/x5WV1cLNJQbcD6+j4epwNfObSS+Y8WfAFucf1s57QWYhe98QBqQ7HUfatGX2U7WVOeHq32lde5z+rIJGOF1HyrlSgK+cjKnAw86y7vjK4IZwFtAuLM8wnmc4Tzf3es+1KIvS5z9kg68wrdXmvntz1ilPl3Mt1deBdw+OUlfAm6fOP//a53bOuA+Z3m9v4fZN/mNMca4wg6RGWOMcYUVGGOMMa6wAmOMMcYVVmCMMca4wgqMMcYYV1iBMcYY4worMMZ4SEQ6iMicWrQ7coLl/xKRCdU9Z4zXrMAY4yFV3a2qnhSISsO5GOMKKzDG1EBEuolv0rbnxDdB2IfOEC/Vtf1MRGaIb0KxzSJygbM8WESeFJFVzsi8t1XadrpzP0pE3nSef0N8k3IlV9r2Y+KbpGyFiLSt9LKXishS5/WuctpGiMiL4pt06isRucRZ/jMReUtE3sU32m57EflCfBNTpR/Pa0xdsAJjTO30BGapaj8gDxh/krYhqjoY+C3wkLPsF/jGeDoL38i8tzjjvVV2O3BIVZOAR4BBlZ6LBlao6gDgC+CWSs91Ay7CN5fJ30UkArgDQFUTgeuAl5zlAOcAN6nqUOB6YLH6RnYegG84IWPqhH1ENqZ2tqvq8Tff1fje1E9kXjXtLgeSKp0vaY6vaG2utN75wF8AVDVdRFIrPVcCLKy03csqPfem+kbG3iIi2/ANL38+8H/OtjaKSCbQy2n/kaoeH8V4FfCCM+r2/Ep9NOa02ScYY2qnuNL9ck7+x1lxNe0E+LWqnuHc4lT1wyrrnWx2x1L9duDAqq9fdUBBrWFbhd80VP0CuBDYBcwWkZ+eZD1jfhArMMbUj8XAr5xPCohIL2co9cr+A/zIeT4BSKzltq8RkSAR6YFvJN1N+A6j/eT4awFdnOXfISJd8U209Ry+qR0G/tCOGXMidojMmPrxT3yHy9Y4c4vk8u2c6Mf9Dd+5klS+Hc4/vxbb3gR8DrQF/kdVi0Tkb/jOx6QBZcDPVLXY99LfcTFwr4iUAkcA+wRj6owN12+MnxCRYCDUKRA98M3Z0UtVSzyOZswpsU8wxviPKOBT5zCaAL+y4mICmX2CMeYUiMgsfPO4V/YXVX3RizzG+CMrMMYYY1xhV5EZY4xxhRUYY4wxrrACY4wxxhVWYIwxxrjCCowxxhhX/D/NIbpFy1UA0gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "# KNN classification (tune k)\n",
    "n_neighbors_list = [201,301,401,501]\n",
    "knn_accuracy_temp = []\n",
    "for n_neighbors in n_neighbors_list:\n",
    "    knn_model_temp = KNeighborsClassifier(n_neighbors = n_neighbors).fit(X_train, y_train)\n",
    "    knn_accuracy_temp.append(knn_model_temp.score(X_test, y_test))\n",
    "    \n",
    "plt.plot(n_neighbors_list, knn_accuracy_temp)\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('n_neighbors')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN test accuracy:  0.08398146615919246\n",
      "\n",
      "\n",
      "KNN confusion matrix:\n",
      " [[ 41   4  16 ...   1   0  52]\n",
      " [ 10  28   5 ...   0   0  30]\n",
      " [ 22   6  14 ...   1   0  26]\n",
      " ...\n",
      " [  9   8   8 ...   5   0  27]\n",
      " [ 11   2   3 ...   2   0  35]\n",
      " [ 22   6   4 ...   2   0 231]]\n"
     ]
    }
   ],
   "source": [
    "n_neighbors = n_neighbors_list[knn_accuracy_temp.index(max(knn_accuracy_temp))]\n",
    "knn_model = KNeighborsClassifier(n_neighbors = n_neighbors, n_jobs = -1).fit(X_train, y_train) # Using all processors\n",
    "knn_predictions = knn_model.predict(X_test)\n",
    "knn_accuracy = knn_model.score(X_test, y_test)\n",
    "knn_cm = confusion_matrix(y_test, knn_predictions)\n",
    "print('KNN test accuracy: ', knn_accuracy)\n",
    "print('\\n')\n",
    "print('KNN confusion matrix:\\n', knn_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Important features for individual occupation clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "929\n",
      "1171\n",
      "746\n",
      "245\n",
      "237\n",
      "195\n"
     ]
    }
   ],
   "source": [
    "# Cluster 20. Typical words: 'founderceo', 'founderandceo', 'cofoundercto'.\n",
    "# Cluster 21. Typical words: 'businessanalyst', 'financemanager', 'businessdevelopmentexecutive'.\n",
    "# Cluster 22. Typical words: 'seniordeveloper', 'softwarearchitect', 'frontenddeveloper'.\n",
    "# Assign binary label columns\n",
    "\n",
    "y_train_19 = [0] * len(y_train)\n",
    "y_train_20 = [0] * len(y_train)\n",
    "y_train_21 = [0] * len(y_train)\n",
    "y_test_19 = [0] * len(y_test)\n",
    "y_test_20 = [0] * len(y_test)\n",
    "y_test_21 = [0] * len(y_test)\n",
    "\n",
    "for i in range(len(y_train)):\n",
    "    if y_train[i] == '19':\n",
    "        y_train_19[i] = 1\n",
    "print(np.sum(y_train_19))\n",
    "\n",
    "for i in range(len(y_train)):\n",
    "    if y_train[i] == '20':\n",
    "        y_train_20[i] = 1\n",
    "print(np.sum(y_train_20))\n",
    "\n",
    "for i in range(len(y_train)):\n",
    "    if y_train[i] == '21':\n",
    "        y_train_21[i] = 1\n",
    "print(np.sum(y_train_21))\n",
    "\n",
    "for i in range(len(y_test)):\n",
    "    if y_test[i] == '19':\n",
    "        y_test_19[i] = 1\n",
    "print(np.sum(y_test_19))\n",
    "\n",
    "for i in range(len(y_test)):\n",
    "    if y_test[i] == '20':\n",
    "        y_test_20[i] = 1\n",
    "print(np.sum(y_test_20))\n",
    "\n",
    "for i in range(len(y_test)):\n",
    "    if y_test[i] == '21':\n",
    "        y_test_21[i] = 1\n",
    "print(np.sum(y_test_21))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline prediction for all the test set:  0\n",
      "Baseline test accuracy:  0.9803905345027304\n"
     ]
    }
   ],
   "source": [
    "# Now try cluster 21\n",
    "# Baseline\n",
    "base_pred = mode(y_train_20)\n",
    "print('Baseline prediction for all the test set: ', base_pred)\n",
    "\n",
    "s = 0\n",
    "for i in range(len(y_test_20)):\n",
    "    if int(y_test_20[i]) == int(base_pred):\n",
    "        s += 1\n",
    "print('Baseline test accuracy: ', s/len(y_test_20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'max_depth')"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8FfX1//HXOyEhCUtYEhYBAVlENlkCLq0iWBWqFRFKwaWiVVuV2tZq1a/9daG1ttXWalVarSLWurUuRQVREfeNsG8CKaCENQiENRDI+f0xE7yEADeQm3uTnOfjcR+Z+5nPzJxBzGE+M3M+MjOcc865ypYU7wCcc87VTJ5gnHPOxYQnGOecczHhCcY551xMeIJxzjkXE55gnHPOxYQnGOecczHhCcY551xMeIJxzjkXE3XiHUA8ZWVlWbt27eIdhnPOVSszZ87caGbZR+pXqxNMu3btyM3NjXcYzjlXrUj6PJp+PkTmnHMuJjzBOOeciwlPMM4552LCE4xzzrmY8ATjnHMuJjzBOOeciwlPMM4552LCE8xR2FZUzENv51FS4tNNO+fcoXiCOQpvLFrPH19bwgPT8+IdinPOJSxPMEdhWO9WDOvdinvfXMp7ywriHY5zziUkTzBHQRJ3DutOp2b1ufHp2azZsiveITnnXMLxBHOUMlLrMP6yvhTvM67/1yz27C2Jd0jOOZdQYppgJA2WtERSnqTbylnfVtI0SfMkvS2pddg+UNKciE+RpIvCdYMkzZK0QNJESXXC9rMkFUZs84tYnhtAh+z6/HFET+as2sLvJi+O9eGcc65aiVmCkZQMPAgMAboCoyV1LdPtHuAJM+sJjAPuAjCz6WbWy8x6AYOAncDrkpKAicAoM+sOfA5cEbG/90q3M7NxsTq3SN/s0ZLvfb09j3+4kklz11TFIZ1zrlqI5RVMfyDPzJab2R7gGWBomT5dgWnh8vRy1gOMAKaY2U6gKbDbzJaG694Ahld65BV025Au5LRtzG3PzyNvw7Z4h+OccwkhlgmmFbAq4nt+2BZpLl8liGFAA0lNy/QZBTwdLm8EUiTlhN9HAG0i+p4maa6kKZK6HesJRCslOYkHLulDRmoyP3hyFjt2762qQzvnXMKKZYJROW1l30y8GRggaTYwAFgN7P/tLKkl0AOYCmBmRpBw7pX0KbAtov8soK2ZnQz8FXip3KCkayXlSsotKKi8R4xbZKZx/6jeLC/Yzu0vzCcI1Tnnaq9YJph8Dry6aA0ccJPCzNaY2cVm1hu4I2wrjOgyEnjRzIojtvnIzM4ws/7Au8CysH2rmW0PlycTXOlklQ3KzB42sxwzy8nOPuKMnxVyescsfnruiUyau4Z/fhzVhG/OOVdjxTLBzAA6SWovKZXgymNSZAdJWeGNe4DbgcfK7GM0Xw2PlW7TLPxZF7gV+Fv4vYUkhcv9Cc7ty0o9oyhcN6ADZ3dpxm9eWcTsLzZX9eGdcy5hxCzBmNleYCzB8NZi4DkzWyhpnKQLw25nAUskLQWaA3eWbi+pHcEV0Dtldn2LpMXAPOBlM3srbB8BLJA0F7if4EmzKh+nSkoSfx7Zi+YN07jhX7PYtGNPVYfgnHMJQbX5XkFOTo7l5ubGZN/z8wsZPv5DTjmhCY9f2Z/kpPJuSTnnXPUjaaaZ5Rypn7/JHyM9Wmfyqwu78d6yjdw/bVm8w3HOuSrnCSaGRvdvw8V9WnH/W8t4e8mGeIfjnHNVyhNMDEnizot6cGLzBvz42Tms9qKYzrlaxBNMjKWnJjP+sr7sC4ti7t67L94hOedclfAEUwXaZ9Xj7m/3ZO6qLdz5qhfFdM7VDp5gqsjg7i255oz2PPHR5/x3zup4h+OcczHnCaYK/WxwF/q1a8xtz89n2Xoviumcq9k8wVSh0qKY9erW4QdPzmS7F8V0ztVgnmCqWPOGafx1dG9WbNzBbc/P86KYzrkayxNMHJzWoSk3n3cir8xby8QPV8Y7HOeciwlPMHHygzM78I2TmvHbVxcz83Mviumcq3k8wcRJUpL407d70bJRGmOfmsWX23fHOyTnnKtUnmDiKDMjhfGX9uXLHXv40TNz2Ffi92OcczWHJ5g4694qk3EXduP9vI3c9+bSeIfjnHOVxhNMAvhOvzaM6Nua+9/KY7oXxXTO1RCeYBKAJH4ztDsntWzIT56dQ/7mnfEOyTnnjllME4ykwZKWSMqTdFs569tKmiZpnqS3JbUO2wdKmhPxKZJ0UbhukKRZkhZImiipTtguSfeHx5onqU8sz62ypacmM/7SPl4U0zlXY8QswUhKBh4EhgBdgdGSupbpdg/whJn1BMYBdwGY2XQz62VmvYBBwE7gdUlJwESC6ZC7A58DV4T7GgJ0Cj/XAuNjdW6x0i6rHveMPJl5+YX85pVF8Q7HOeeOSSyvYPoDeWa23Mz2AM8AQ8v06QpMC5enl7MeYAQwxcx2Ak2B3WZWejf8DWB4uDyUIFmZmX0MNJLUsvJOp2qc160F3z/zBJ78+Atemu1FMZ1z1VcsE0wrYFXE9/ywLdJcvkoQw4AGkpqW6TMKeDpc3gikSCqdC3oE0KYCx0PStZJyJeUWFBRU4HSqzi3nnUj/9k24/YX5LFnnRTGdc9VTLBOMymkr+6LHzcAASbOBAcBqYH8FyPAKpAcwFcCCwl2jgHslfQpsi+gfzfEws4fNLMfMcrKzsyt2RlWkTnISD4zuTb26dbjuyZlsKyqOd0jOOVdhsUww+Xx1dQHQGlgT2cHM1pjZxWbWG7gjbCuM6DISeNHMiiO2+cjMzjCz/sC7wLJoj1edNGuYxgOX9ObzTTu51YtiOueqoVgmmBlAJ0ntJaUSXHlMiuwgKSu8cQ9wO/BYmX2M5qvhsdJtmoU/6wK3An8LV00Cvhs+TXYqUGhmayvzhKraqSc05ZbzTmTy/HU89sHKeIfjnHMVErMEY2Z7gbEEw1uLgefMbKGkcZIuDLudBSyRtBRoDtxZur2kdgRXJO+U2fUtkhYD84CXzeytsH0ysBzIAx4Bro/BaVW57595Aud0bc5dkxeTu3JTvMNxzrmoqTYPveTk5Fhubm68wziiwl3FXPjA+xQV7+PVG88gq37deIfknKvFJM00s5wj9fM3+auBzPQUHrq0D1t2FvOjZ2Z7UUznXLXgCaaa6HZcJr8Z2p0P8r7k3je8KKZzLvF5gqlGRvZrw8ic1jwwPY+3Plsf73Ccc+6wPMFUM+OGdqdry4b85Nm5rNrkRTGdc4nLE0w1k5aSzN8u60uJBUUxi4q9KKZzLjF5gqmGjm+awZ9H9mL+6kLGeVFM51yC8gRTTZ3TtTk/GNCBpz75gudn5sc7HOecO4gnmGrs5nM7c+oJTbjjpfl8tm5rvMNxzrkDeIKpxuokJ3H/6N40TEvhuidnsdWLYjrnEognmGquWYM0HrikD19s2snP/u1FMZ1zicMTTA3Qv30Tbh18Iq8tXMej76+IdzjOOQdAnXgH4CrHNWecwMzPN/PbVxfzj/dW0CIzjRYN02iRmUbLzLT931tmptM8sy516yTHO2TnXA3nCaaGkMSfRvaixwcrWPnlTtZvLSKvYDvvLStgx56D35VpWi+V5g2/Sj4tM9PC7+lBMspMo35d/+vhnDt6/hukBqlftw5jB3U6qH1bUTHrtxaxtjD4rCssYt3W4OeawiJmr9rCph17DtquQd06+5NNi/3JKD0iGaXRKCMFqbzJRJ1ztZ0nmFqgQVoKDdJS6NiswSH7FBXv25+ESn+uKyxibeEu1m3dzdL1BWzYtpuyzxDUrZN0UAJq0bDu/kTUIjONrPp1SU7yJORcbeMJxgFBCZq2TevRtmm9Q/Yp3ldCwbbd+69+DkxGu8j9fDPrt66leN+BWSg5SbRqlM53T2vL5ae19fs/ztUSMU0wkgYD9wHJwD/M7Pdl1rclmCY5G9gEXGZm+ZIGAvdGdO0CjDKzlySdDdxN8ATcdmCMmeVJGhO2rw63ecDM/hG7s6t9UpKTOK5ROsc1Sj9kn5ISY9POPfsTUJCMdjHr8y389tXFTPhgJT89tzMX9WpFkl/VOFejxWxGS0nJwFLgHCAfmAGMNrNFEX3+DbxiZhMlDQKuNLPLy+ynCcE0yK3NbGc4vfJQM1ss6Xqgv5mNCRNMjpmNjTbG6jKjZU3x/rKN3DVlMQvXbOWklg25dfCJDOic7fdwnKtmEmFGy/5AnpktN7M9wDPA0DJ9ugLTwuXp5awHGAFMMbPS2vQGNAyXM4E1lRq1i5mvd8ri5bFf575Rvdi+u5gxE2Zw6T8+YV7+lniH5pyLgVgmmFbAqojv+WFbpLnA8HB5GNBAUtMyfUYBT0d8vxqYLCkfuByIHHYbLmmepP9IalNeUJKulZQrKbegoKBiZ+SOWVKSGNqrFW/eNIBffqsrn63bxoUPfMDYp2bx+Zc74h2ec64SxTLBlDfuUXY87mZggKTZwACC+yd79+9Aagn0AKZGbPMT4Jtm1hqYAPw5bH8ZaGdmPYE3gYnlBWVmD5tZjpnlZGdnV/ysXKWoWyeZK7/WnnduOYsfDurItMUbOPtP7/DL/y5g4/bd8Q7POVcJYplg8oHIq4jWlBnOMrM1ZnaxmfUG7gjbCiO6jAReNLNiAEnZwMlm9km4/lng9HC7L82s9DfTI0DfSj4fFwMN0lL46bkn8s4tZzGyXxue/OQLBvxxOve9uYwdu/ceeQfOuYQVywQzA+gkqb2kVIKhrkmRHSRlSSqN4XaCJ8oijebA4bHNQKakzuH3c4DF4b5aRvS7sLTdVQ/NGqbxu2E9mPrjM/l6pyzufXMpA+5+m39+/DnF+0riHZ5z7ijELMGY2V5gLMHw1mLgOTNbKGmcpAvDbmcBS8Inw5oDd5ZuL6kdwRXQO2X2eQ3wvKS5BPdgbglX3yhpYdh+IzAmVufmYqdjs/r8/fIcnr/udNpnZfD/XlrAufe+y+T5a71StHPVTMweU64O/DHlxGZmTFu8gT+89hnLNmynV5tG3D6kC6ecUPY5EOdcVUqEx5SdOyaS+EbX5rz24zP54/CerCss4jsPf8xVj89gybpt8Q7POXcEfgXjVzDVRlHxPiZ8sJKH3s5j++69DO/TmpvO6XzYygLOucoX7RWMJxhPMNXOlp17eHB6HhM//BwEV57ejuvP6khmRkq8Q3OuVvAEEwVPMNVb/uad/PmNpbw4ezUN6tbhhoEdueL0dqSleDFN52LJE0wUPMHUDIvWbOWPUz/j7SUFHJeZxk3nnsiw3q18igDnYsRv8rtao+txDXn8yv48dc0pZDWoy83/nsv597/H9M82+KPNzsWRJxhXY5zeIYv/3vA1HrikN7uK93Hl4zMY9fDHzFnlxTSdiwdPMK5GkcQFPY/jjZ8MYNzQbuRt2M5FD37A9f+ayYqNXkzTuark92D8HkyNtn33Xh55dzmPvLecPXtLGNW/DT86uzPZDerGOzTnqi2/yR8FTzC1x4ZtRfx1Wh5Pf/oFqXWSuPqME7j2zBOoX9dnDXeuojzBRMETTO2zYuMO7pm6hFfnryWrfio3nt2JS09p60+cOVcB/hSZc+Von1WPBy/tw0s3fI0O2fX5xX8XcseL8/1pM+diwBOMq5V6tWnEM9eeytiBHXlmxir+8NqSeIfkXI3jA9Cu1pLET8/tzOade/jbO/+jcUYK3x/QId5hOVdjeIJxtZokxg3tTuGuYu6a8hmNM1IZ2a/NkTd0zh2RJxhX6yUniT+P7MXWor3c9sI8GqanMLh7i3iH5Vy1F9U9GEnPSzo/YnrjqEgaLGmJpDxJt5Wzvq2kaZLmSXpbUuuwfaCkORGfIkkXhevOljQrbH9fUsewva6kZ8NjfRLOiOlcVFLrJPG3y/pwcptG3Pj0bD7M2xjvkJyr9qJNGOOBS4Blkn4vqcuRNpCUDDwIDAG6AqMldS3T7R7gCTPrCYwD7gIws+lm1svMegGDgJ3A6xGxXBquewr4edj+PWCzmXUE7gX+EOW5OQdARmodJozpR/uselzzRC7z8r3EjHPHIqoEY2ZvmtmlQB9gJfCGpA8lXSnpUJNw9AfyzGy5me0BngGGlunTFZgWLk8vZz3ACGCKme0sDQdoGC5nAmvC5aHAxHD5P8DZkvzlBlchjTJSeeJ7/WlcL5UxE2aQt2F7vENyrtqKeshLUlNgDHA1MBu4jyDhvHGITVoBqyK+54dtkeYCw8PlYUCD8DiRRgFPR3y/GpgsKR+4HPh92eOZ2V6gEDho8nZJ10rKlZRbUFBwiNBdbda8YRpPfu8UkiQuf/QTVm/ZFe+QnKuWor0H8wLwHpABfMvMLjSzZ83sh0D9Q21WTlvZt9luBgZImg0MAFYDeyOO2xLoAUyN2OYnwDfNrDUwAfhzBY6HmT1sZjlmlpOdnX2I0F1t1y6rHk9c1Z/tu/dy+aOf8OX23fEOyblqJ9ormAfMrKuZ3WVmayNXHKZcQD4Q+bxna74azirddo2ZXWxmvYE7wrbCiC4jgRfNrBhAUjZwspl9Eq5/Fji97PEk1SEYPtsU5fk5d5CuxzXk0Sv6sXrzLsZMmMH23XuPvJFzbr9oE8xJkhqVfpHUWNL1R9hmBtBJUntJqQRDXZMiO0jKingy7XbgsTL7GM2Bw2ObgUxJncPv5wCLw+VJwBXh8gjgLfP6H+4Y9W/fhPGX9WHR2q1cMzGXouJ98Q7JuWoj2gRzjZntf6TGzDYD1xxug/A+yFiC4a3FwHNmtlDSOEkXht3OApZIWgo0B+4s3T58zLgN8E6ZfV4DPC9pLsE9mFvC1Y8CTSXlATcBBz0W7dzRGNSlOX/69sl8tPxLbnx6Nnv3lcQ7JOeqhaiqKUuaRzA0ZeH3ZGCemXWLcXwx5dWUXUVM+GAFv355ESNzWvOH4T3xhxRdbRVtNeVo3+SfCjwn6W8EN85/ALx2DPE5V+1c+bX2bN5ZzP3TltEoI5Xbh3TxJOPcYUSbYG4Fvg9cR/C01uvAP2IVlHOJ6iff6MSWnXt4+N3lNM5I5bqzvDimc4cSVYIxsxKCN+jHxzYc5xKbJH71rW5s2VnMH177jEYZKYzuf3y8w3IuIUWVYCR1Iijj0hVIK203sxNiFJdzCSspSdzz7ZPZWlTMHS/OJzM9hW/2aBnvsJxLONE+RTaB4OplLzAQeAL4Z6yCci7RpdZJYvylfel9fGN+/Mwc3l/mxTGdKyvaBJNuZtMInjr73Mx+RVCE0rlaKz01mceu6McJ2fW49p+5zFnlxTGdixRtgikKX4hcJmmspGFAsxjG5Vy1kJmRwhNX9Serfl3GTPiUZeu3xTsk5xJGtAnmxwR1yG4E+gKX8dVb887Vas3C4pgpyUlc/uin5G/eeeSNnKsFjphgwpcqR5rZdjPLN7MrzWy4mX1cBfE5Vy0c3zSDJ67qz849e7n80U/Z6MUxnTtygjGzfUBfn1vFucM7qWVDHhvTj7WFu7jisU/ZVlQc75Cci6toh8hmA/+VdLmki0s/sQzMueoop10Txl/WlyXrtnG1F8d0tVy0CaYJ8CXBk2PfCj8XxCoo56qzgSc2408jT+bTlZsY+5QXx3S1V7Rv8l8Z60Ccq0mG9mpF4a5ifvHfhdz6/HzuHtGTpCQfZXa1S7Rv8k+g/Nkhr6r0iJyrIb57Wjs27yjm3jeX0igjhZ+ff5IXx3S1SrTFLl+JWE4DhlFmdkrn3MFuPLsjm3fu4dH3V9CkXio3DOwY75CcqzLRDpE9H/ld0tPAm0faTtJg4D4gGfiHmf2+zPq2BLNYZhNMb3yZmeVLGgjcG9G1CzDKzF6S9B7QIGxvBnxqZhdJOgv4L7AiXPeCmY2L5vycixVJ/OKCrmzZuYe7py6hUUYKl57SNt5hOVclor2CKasTcNgSsuH7Mw8STGucD8yQNMnMFkV0uwd4wswmShpEUFDzcjObDvQK99MEyCOYIgAzOyPiGM8TJJVS75mZP3zgEkpSkrj72yeztWgvP39pAZnpKVzQ87h4h+VczEX1FJmkbZK2ln6AlwnmiDmc/kCemS03sz3AM8DQMn26AtPC5enlrAcYAUwxswNej5bUgOCptpeiOQfn4iklOYkHL+lDTtvG/OTZOby7tCDeITkXc1ElGDNrYGYNIz6dyw6blaMVsCrie37YFmkuMDxcHgY0kNS0TJ9RwNPl7H8YMM3Mtka0nSZprqQpkqr1dM6u5klPTeYfV/SjY7MGfP+fM5n1xeZ4h+RcTEV7BTNMUmbE90aSLjrSZuW0lX0S7WZggKTZwABgNcGUAKXHaQn0IJiyuazRHJh4ZgFtzexk4K8c4spG0rWSciXlFhT4vyJd1cpMT2HiVf1o1rAuV06YwVIvjulqsGhftPylmRWWfjGzLcAvj7BNPtAm4ntryjx5ZmZrzOxiM+sN3BG2FUZ0GQm8aGYH1NwIr3L6A69G7GurmW0PlycDKZKyygZlZg+bWY6Z5WRnZx/hFJyrfM0aBMUx69ZJ4vJHP2HVJi+O6WqmaBNMef2O9IDADKCTpPaSUgmGuiZFdpCUFU4DAHA7wRNlkcpepZT6NvCKmRVF7KtFab00Sf3DmL88QozOxUWbJhn883unUFRcwuWPfkLBNi+O6WqeaBNMrqQ/S+og6QRJ9wIzD7eBme0FxhIMby0GnjOzhZLGSbow7HYWsETSUqA5cGfp9pLaEVwBvVPO7su7LzMCWCBpLnA/wWPNB70c6lyiOLFFAx4b04/1W3fz3cc+pXCXF8d0NYui+R0sqR7w/4BvhE2vA3ea2Y4YxhZzOTk5lpubG+8wXC33ztICrp44g95tGjPxqv6kpybHOyTnDkvSTDPLOVK/aJ8i22Fmt5XeuzCz/6vuycW5RDGgczZ/HtmLGZ9vYuxTsyj24piuhoj2KbI3JDWK+N5YUnlPdjnnjsK3Tj6O3wztzrTPNvCz/8yjpMRHd131F+2b/Fnhk2MAmNlmSc1iFJNztdJlp7Zly8493PP6UjLTU/i/b55Eap1ob5M6l3iiTTAlko43sy9g/w14/yeWc5XshoEd2bSjmMc+WMHjH64kq35dWmam0bxhGi0z02iRGf5sGCy3yEwjI/VoKz45F1vR/s28A3hfUukTXWcC18YmJOdqL0n8/PyT6NO2EXkbtrOusIh1W4vI37yTGSs3lfukWWZ6yv6Ec3AySqdFwzQaptfxqQJclYu2mvJrknIIksocggKTu2IZmHO1VVKSDlkMc9eefazbWsTawl37k8+6wiLWFhaxfmsRi9ZuZeP23ZR9ODQ9JXl/0jk4GaXTIjONpvVSfVI0V6minXDsauBHBG/jzwFOBT4iKDbpnKsi6anJtM+qR/useofss2dvCRu2BQlnbWGQgNYVFrE2TEafrNjE+q1F7C3zIEFKsmjW4MChuMgE1CIzjWYN6pKS7PeFXHSiHSL7EdAP+NjMBkrqAvw6dmE5545Wap0kWjfOoHXjjEP2KSkxNu7YvT/5rCuTjBau2cqbi9dTVHzgI9N1ksSPzu7E2EEdfcjNHVG0CabIzIokIamumX0m6cSYRuaci5mkpOBqpVmDNHq2Lr+PmVG4q/iA5PPu0gL+9MZSPt+0k98N6+FPubnDijbB5IfvwbwEvCFpMz5lsnM1miQaZaTSKCOVLi0aAjCqXxvum7aMv7y5jDVbdjH+sr5kpqfEOVKXqKIqFXPABtIAIBN4LZxIrNryUjHOHZ3nZ+Zz2wvzaNe0HhOu7HfY4ThX81RqqZhIZvaOmU2q7snFOXf0hvdtzcSr+rNuaxHDHvqQeflbjryRq3V8ANU5d1RO75DFC9edTmpyEt/5+8e8uWh9vENyCcYTjHPuqHVq3oAXbzidzs3rc+0/c3n8gxXxDsklEE8wzrlj0qxBGs9cexrfOKk5v3p5EeNeXsQ+L9bp8ATjnKsE6anJjL+sL1d9rT2PfbCC656cya49++IdlouzmCYYSYMlLZGUJ+m2cta3lTRN0jxJb0tqHbYPlDQn4lMk6aJw3XsR7WskvRS2S9L94bHmSeoTy3Nzzh0oOUn84ltd+dW3uvLm4vWMevgjnwq6lotZgpGUDDwIDAG6AqMldS3T7R7gCTPrCYwD7gIws+lm1svMehGUo9lJMIsmZnZGxLqPgBfCfQ0BOoWfa4HxsTo359yhjflae/5+eQ5L129n2EMfsGz9tniH5OIkllcw/YE8M1sePtL8DDC0TJ+uwLRweXo56wFGAFPMbGdko6QGBMnnpbBpKEGyMjP7GGgkqWXlnIpzriLO6dqcZ79/KkXFJVw8/kM+/N/GeIfk4iCWCaYVsCrie37YFmkuMDxcHgY0kNS0TJ9RwNPl7H8YMM3MtlbgeM65KtKzdSNeuuF0WmamccVjn/L8zPx4h+SqWCwTTHmV8Mo+WnIzMEDSbGAAsBrYu38HwRVID6C86ZlHc2DiieZ4SLpWUq6k3IKCgsOfgXPumLRunMG/f3A6/do14af/nsu9byylotVDXPUVywSTD7SJ+N6aMvXLzGyNmV1sZr0JJjXDzAojuowEXjSzA2ZZCq9y+gOvVuR44f4fNrMcM8vJzs6u+Fk55yokMz2Fx6/sz4i+rblv2jJ++txc9uwtOfKGrtqLZYKZAXSS1F5SKsFQ16TIDpKyJJXGcDvwWJl9lL1KKfVt4BUzK4pomwR8N3ya7FSg0MzWVsaJOOeOTWqdJO4e0ZObzunMC7NXc8Vjn1K48+DZOV3NErMEY2Z7gbEEw1uLgefMbKGkcZIuDLudBSyRtBRoDtxZur2kdgRXJO9wsPLuy0wGlgN5wCPA9ZV1Ls65YyeJG8/uxL3fOZnczzdx8fgPWLVp55E3dNVWhasp1yReTdm5+Ph4+Zd8/58zSUkW/7iiH73aNIp3SK4CYlZN2TnnjtWpJzTl+etOJz01mVEPf8TUheviHZKLAU8wzrm46NisPi9e/zW6tGjID56cyaPve6HMmsYTjHMubrLq1+Xpa07lvK4t+M0ri/jVpIU1plBm4a5iXluwls07au/UWdFOmeycczGRnprMQ5f24a4pi3nkvRXkb97J/aN7k5Fa/X49bdqxhzcWrWPy/HV8+L+NFO8zzuiUxRNX9Ucq71W9mq36/Rd0ztU4SUnijvNcsnAbAAAU60lEQVS70qZJBr+atJDv/P1jHr0ih2YN0+Id2hFt2FrE1IXrmLJgHZ+s2MS+EqN143TGnN6OlOQkHnr7fzw7YxWj+h8f71CrnCcY51zC+O5p7WjVKJ2xT81m2EMfMuHKfnRu3iDeYR1k9ZZdvLZgHVPmr2XmF5sxgxOy6/GDAScwpHtLuh3XEEmUlBizvtjMna8uZsCJ2bTMTI936FXKH1P2x5SdSzgLVhdy1eMz2LVnH+Mv68vXO2XFOyRWbtzBlAXreG3BWubmBwVHurRowJDuLRnSowWdmtUvdxjsiy93ct5f3uWUE5owYUy/GjFUFu1jyp5gPME4l5BWb9nFVRNm8L+C7fzu4h6MzGlz5I0q2bL125iyYB2T56/ls3XBtAM9W2cyuHsLhnRvSfuselHtZ8IHK/j1y4u459snM6Jv61iGXCU8wUTBE4xziW1rUTE3/GsW7y3byA8HdeSmczrH9ArAzFi4Zmsw/LVgLf8r2IEEfY9vzODuLRjcvQWtG2dUeL8lJcZ3Hv6IJeu28cZNA2heDe4tHY4nmCh4gnEu8RXvK+HnLy7g2dxVXNTrOP4woid16yRX2v5LSoy5+VvC4a91fLFpJ0kKXgYd0r0F53VrUSkPG6zYuIPBf3mXMzpl88h3+1brobJoE4zf5HfOJbSU5CR+P7wHxzfN4O6pS1hTWMTDl/elUUbqUe9zX4mRu3ITUxasY+rCdawtLCIlWZzeIYvrz+rAOV2b07R+3Uo8C2ifVY+bzz2ROycvZtLcNQztVfOnq/IrGL+Cca7a+O+c1dzy73m0bpLO42P6c3zT6IeriveV8MnyTUxesJbXF65n4/bdpNZJYkDnbIZ0b8HZJzUnMz0lhtEHiW34+A9Z+eUO3vjJALIbVG4Sqyo+RBYFTzDOVT+frtjEtf/MJVnikSty6HN840P23b13Hx/kbWTK/HW8sXg9W3YWk5GazMATmzGkRwsGntiMenWrdiBn2fptnH//+5x9UjPGX9a3So9dWTzBRMETjHPV0/8KtnPlhBms31rEX77TiyE9Wu5ft2vPPt5ZWsCUBWt5a/EGtu3eS4O0OnzjpOYM7t6CAZ2zSUupvHs4R+PB6XncPXUJD17Sh/N7tjzyBgnGE0wUPME4V319uX03Vz+Ry5xVW7h1cBeOa5TOawvWMv2zAnYV76NxRgrndm3B4B4t+FqHLFLrJE7pxb37Shj20Ies2bKLN24aQJN6R38/KR48wUTBE4xz1VtR8T5uem4Ok+cH5f6zG9TlvG7N+Wb3lvRv34Q6yYmTVMr6bN1WvvXX9xnSvSX3j+4d73AqJCGeIpM0GLgPSAb+YWa/L7O+LcE0ydnAJuAyM8uXNBC4N6JrF2CUmb2k4Nm+3xJMm7wPGG9m90s6C/gvUFrz+wUzGxe7s3POxVtaSjIPjO7D1J7ryGpQl77HNyYpqXo8/tulRUPGDuzEvW8u5YKeLTm3W4t4h1TpYpZgJCUDDwLnAPnADEmTzGxRRLd7gCfMbKKkQcBdwOVmNh3oFe6nCcE0yK+H24whmEq5i5mVSGoWsb/3zOyCWJ2Tcy7xJCXpgHsw1cn1Azvw2sJ13PHSAvq3b3JMj14nolheP/YH8sxsuZntAZ4Bhpbp0xWYFi5PL2c9wAhgipmVTt59HTDOzEoAzGxDpUfunHNVICU5ibtH9GTzjj2Me2XRkTeoZmKZYFoBqyK+54dtkeYCw8PlYUADSU3L9BkFPB3xvQPwHUm5kqZI6hSx7jRJc8P2bsd+Cs45F1vdW2Vy3VkdeGHWat76bH28w6lUsUww5Q2Eln2i4GZggKTZwABgNbB3/w6klkAPYGrENnWBovAG0yME93AAZgFtzexk4K/AS+UGJV0bJqfcgoKCip+Vc85VsrGDOtK5eX3+74UFbC0qjnc4lSaWCSaf4F5JqdbAmsgOZrbGzC42s97AHWFbYUSXkcCLZhb5J54PPB8uvwj0DLfbambbw+XJQIqkg2p8m9nDZpZjZjnZ2dnHdILOOVcZ6tZJ5u4RJ7NhWxF3vrI43uFUmlgmmBlAJ0ntJaUSDHVNiuwgKUtSaQy389XVSKnRHDg8BsGVyaBweQCwNNxXi/AJMyT1Jzi3LyvpXJxzLqZObtOIa8/swLO5q3h3ac0YXYlZgjGzvcBYguGtxcBzZrZQ0jhJF4bdzgKWSFoKNAfuLN1eUjuCK6B3yuz698BwSfMJnjq7OmwfASyQNBe4n+Cx5tr7ko9zrtr58Tc60SG7Hre/MJ/tu/ceeYME5y9a+ouWzrkEMvPzzYz424dc0v947hzWI97hlCvaFy0T9zVX55yrhfq2bcxVX2vPvz75gg/zNsY7nGPiCcY55xLMzeeeSLumGdz6wjx2VOOhMk8wzjmXYNJTk/nD8J6s2rSLu6cuiXc4R80TjHPOJaBTTmjKFae1ZeJHK5mxclO8wzkqnmCccy5B/WxwF1o3Tudn/5nHrj374h1OhXmCcc65BFWvbh3+cHFPVmzcwZ/fqH5DZZ5gnHMugZ3eMYtLTjmeR99fwawvNsc7nArxBOOccwnu9iFdaNEwjVv+PZei4uozVOYJxjnnElyDtBTuGt6T/xXs4L5py+IdTtQ8wTjnXDUwoHM2I3Na8/C7y5mXvyXe4UTFE4xzzlUTd5zflaz6qdzy73ns3pv4Q2WeYJxzrprITE/hrot7sGT9Nh58Ky/e4RyRJxjnnKtGBnVpzsW9W/HQ2/9j4ZrCI28QR55gnHOumvnFt7rSuF4wVFa8ryTe4RySJxjnnKtmGmWk8tuLurNo7VbGv/2/eIdzSJ5gnHOuGjqvWwsu6NmSv761jCXrtsU7nHLFNMFIGixpiaQ8SbeVs76tpGmS5kl6W1LrsH2gpDkRnyJJF4XrJOlOSUslLZZ0Y0T7/eGx5knqE8tzc865ePv1hd1omJbCLf+Zy94EHCqLWYKRlAw8CAwBugKjJXUt0+0e4Akz6wmMI5gCGTObbma9zKwXMAjYCbwebjOGYCrlLmZ2EvBM2D4E6BR+rgXGx+jUnHMuITStX5dfD+3GvPxCHnlvRbzDOUgsr2D6A3lmttzM9hAkgqFl+nQFpoXL08tZDzACmGJmO8Pv1wHjzKwEwMw2hO1DCZKVmdnHQCNJLSvvdJxzLvGc36Mlg7u14N43l5K3YXu8wzlALBNMK2BVxPf8sC3SXGB4uDwMaCCpaZk+o4CnI753AL4jKVfSFEmdKnA855yrUSTxm4u6k5GazC3/mcu+Eot3SPvFMsGonLayZ34zMEDSbGAAsBrYPz9oeAXSA5gasU1doMjMcoBHgMcqcDwkXRsmp9yCgoJoz8U55xJWdoO6/Opb3Zj9xRYmfJA4Q2WxTDD5BPdKSrUG1kR2MLM1ZnaxmfUG7gjbIt8cGgm8aGbFZfb7fLj8ItAz2uOF+3/YzHLMLCc7O7viZ+WccwloaK/j+MZJzbh76hJWbNwR73CA2CaYGUAnSe0lpRIMdU2K7CApS1JpDLfz1dVIqdEcODwG8BLBjX8IrnqWhsuTgO+GT5OdChSa2drKORXnnEtskrhzWA/q1kni1v/MoyQBhspilmDMbC8wlmB4azHwnJktlDRO0oVht7OAJZKWAs2BO0u3l9SO4IrknTK7/j0wXNJ8gqfOrg7bJwPLgTyCobPrK/+snHMucTVvmMb/u6Arn67cxBMfrYx3OMgs/lkuXnJyciw3NzfeYTjnXKUxM658fAafLN/E1B+fyfFNMyr9GJJmhvfBD8vf5HfOuRpEEr8b1oPkJHHr8/EdKvME45xzNcxxjdK54/yT+Gj5lzz16Rdxi8MTjHPO1UCj+rXh6x2zuGvyYvI37zzyBjHgCcY552ogSdx1cQ8MuP2F+cTjfrsnGOecq6HaNMngtiFdeG/ZRv6dm1/lx/cE45xzNdhlp7TllPZN+M2ri1hXWFSlx/YE45xzNVhSkvjD8J4U7yvh/16s2qEyTzDOOVfDtcuqxy3ndeGtzzbw4uzVVXZcTzDOOVcLjDm9HX3bNubXLy9iw9aqGSrzBOOcc7VAcpL444ieFBXv4+cvLaiSoTJPMM45V0t0yK7PTed05vVF63l5XuxrAXuCcc65WuTqM07gtBOasnP33iN3PkZ1Yn4E55xzCSM5STx1zSlI5c3RWLn8CsY552qZqkgu4AnGOedcjHiCcc45FxMxTTCSBktaIilP0m3lrG8raZqkeZLeltQ6bB8oaU7Ep0jSReG6xyWtiFjXK2w/S1JhRPsvYnluzjnnDi9mN/klJQMPAucA+cAMSZPMbFFEt3uAJ8xsoqRBBFMgX25m04HSxNGEYBrk1yO2u8XM/lPOYd8zswticDrOOecqKJZXMP2BPDNbbmZ7gGeAoWX6dAWmhcvTy1kPMAKYYmbxmdDAOefcUYllgmkFrIr4nh+2RZoLDA+XhwENJDUt02cU8HSZtjvDYbV7JdWNaD9N0lxJUyR1O8b4nXPOHYNYJpjynoMrW5vgZmCApNnAAGA1sP/tH0ktgR7A1Ihtbge6AP2AJsCtYfssoK2ZnQz8FXip3KCkayXlSsotKCio8Ek555yLTixftMwH2kR8bw2siexgZmuAiwEk1QeGm1lhRJeRwItmVhyxTWl9g92SJhAkKcxsa0SfyZIekpRlZhvLHPNh4OHwmAWSPj/K88sCNh6xV9VL1LggcWPzuCrG46qYmhhX22g6xTLBzAA6SWpPcGUyCrgksoOkLGCTmZUQXJk8VmYfo8P2yG1amtlaBW8KXQQsCNtbAOvNzCT1J7g6+/JwAZpZ9tGenKRcM8s52u1jJVHjgsSNzeOqGI+rYmpzXDFLMGa2V9JYguGtZOAxM1soaRyQa2aTgLOAuyQZ8C5wQ+n2ktoRXAG9U2bX/5KUTTAENwf4Qdg+ArhO0l5gFzDK4jEJtXPOOSDGtcjMbDIwuUzbLyKW/wOU97gxZraSgx8KwMwGHaL/A8ADxxCuc865SuRv8h+9h+MdwCEkalyQuLF5XBXjcVVMrY1LPorknHMuFvwKxjnnXEx4gjkKkn4iaaGkBZKelpQW75gAJP0ojGmhpB/HMY7HJG2QtCCirYmkNyQtC382TpC4vh3+eZVIisuTPoeI625Jn4UvFL8oqVGCxPWbMKY5kl6XdFxVx3Wo2CLW3SzJwqdU4xqTpF9JWh1RI/GbVRnT4WIL238Y1otcKOmPlX1cTzAVJKkVcCOQY2bdCZ6QGxXfqEBSd+AaghI9JwMXSOoUp3AeBwaXabsNmGZmnQjKAx1U/LQKPM7BcS0geBfr3SqP5iuPc3BcbwDdzawnsJQyj+tXkcc5OK67zaynmfUCXgHiVVT2cQ6ODUltCOofflHVAXGImIB7zaxX+Jlczvqq8DhlYpM0kKA8V08z60ZQG7JSeYI5OnWAdEl1gAzKvEAaJycBH5vZTjPbS/B497B4BGJm7wKbyjQPBSaGyxMJ3mGqUuXFZWaLzWxJVcdSJoby4no9/O8I8DHBi8qJENfWiK/1OLg6R5U4xN8xgHuBnxGHuA4TU9wdIrbrgN+b2e6wz4bKPq4nmAoys9UEmf4LYC1QaGavH36rKrEAOFNSU0kZwDc5sJJCvDUvrcIQ/mwW53iqk6uAKfEOopSkOyWtAi4lflcwB5F0IbDazObGO5YyxobDio/FY2j4MDoDZ0j6RNI7kvpV9gE8wVRQ+BdkKNAeOA6oJ+my+EYV/Esc+APB0MprBIVE9x52I5fwJN1B8N/xX/GOpZSZ3WFmbQhiGhvveADCf1TdQQIlvNB4oAPB9CNrgT/FN5wD1AEaA6cCtwDPhRVSKo0nmIr7BrDCzArCGmkvAKfHOSYAzOxRM+tjZmcSXA4vi3dMEdaHxUtLi5hW+uV4TSPpCuAC4NIErUrxFF9VQ4+3DgT/6JsraSXBkOKssIRU3JjZejPbF5bDeoTgHmmiyAdesMCnQAlBfbJK4wmm4r4ATpWUEWb7s4HFcY4JAEnNwp/HE9y4LjvNQTxNAq4Il68A/hvHWBKepMEElcIvTKS5kMo8OHIh8Fm8YolkZvPNrJmZtTOzdgS/PPuY2bp4xlX6j6rQMMLaiQniJWAQgKTOQCqVXZTTzPxTwQ/wa4L/sRYA/wTqxjumMK73gEUEw2NnxzGOpwmGA4oJ/kf/HtCU4OmxZeHPJgkS17BweTewHpiaIHHlEcynNCf8/C1B4no+/Hs/D3gZaJUof8fKrF8JZMU7pvD3w/zwz2sS0DJR/rwIEsqT4X/PWcCgyj6uv8nvnHMuJnyIzDnnXEx4gnHOORcTnmCcc87FhCcY55xzMeEJxjnnXEx4gnHOORcTnmCcqwYkrTza8vOSxkSW1T+WfTlXEZ5gnKv5xhDUzXOuSnmCca4CJLULJwL7Rzi5278kfUPSB+Fkav3Dz4eSZoc/Twy3vUnSY+Fyj3D7jEMcp2k4oddsSX8HFLHuMkmfhhNY/V1Scti+XdKfJM2SNE1StqQRQA7wr7B/eribH4b95kvqEss/M1d7eYJxruI6AvcBPYEuwCXA14Gbgf8jKCN0ppn1Jqju+7twu78AHSUNAyYA37dD1xn7JfB+uI9JwPEAkk4CvgN8zYJJv/YRlM2HYH6WWWbWh2A+oF+a2X+AXIKCmb3MbFfYd2PYb3wYt3OVrk68A3CuGlphZvMBJC0kmKnTJM0H2gGZwMSwMKQBKQBmViJpDEFdqr+b2QeHOcaZBAVLMbNXJW0O288G+gIzwsrq6XxVmboEeDZcfpKg0vehlK6bWXoc5yqbJxjnKm53xHJJxPcSgv+nfgNMN7NhktoBb0f07wRsJ7p7IuUVChQw0cyimUL5cIUGS2Peh/8ecDHiQ2TOVb5MYHW4PKa0UVImwdDamUDT8P7IobxLOPQlaQjBxFAQVKIeETE1QxNJbcN1SUDpPi8B3g+XtwENjuF8nDsqnmCcq3x/BO6S9AGQHNF+L/CQmS0lKJf++9JEUY5fE0yBPQs4l2AeIsxsEfBz4HVJ8whmMC2dc2QH0E3STIJ5PsaF7Y8Dfytzk9+5mPNy/c7VEJK2m1n9eMfhXCm/gnHOORcTfgXjXBxJuhL4UZnmD8zshnjE41xl8gTjnHMuJnyIzDnnXEx4gnHOORcTnmCcc87FhCcY55xzMeEJxjnnXEz8f55ocg4DZeS5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Decision tree classification (tune max_depth)\n",
    "max_depth_list = [8,9,10,11,12,13,14,15,16]\n",
    "dtree_accuracy_temp = []\n",
    "for max_depth in max_depth_list:\n",
    "    dtree_model_temp = DecisionTreeClassifier(max_depth = max_depth).fit(X_train, y_train_20)\n",
    "    dtree_accuracy_temp.append(dtree_model_temp.score(X_test, y_test_20))\n",
    "    \n",
    "plt.plot(max_depth_list, dtree_accuracy_temp)\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('max_depth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision tree test accuracy:  0.9793149098130068\n",
      "\n",
      "\n",
      "Decision tree confusion matrix:\n",
      " [[11836    13]\n",
      " [  237     0]]\n",
      "\n",
      "\n",
      "Decision tree important features:\n",
      " [0.03347879 0.05665834 0.00451102 0.00412099 0.02031457 0.00698005\n",
      " 0.         0.         0.         0.         0.         0.02138396\n",
      " 0.02073293 0.03147236 0.         0.         0.         0.\n",
      " 0.         0.00680844 0.         0.         0.01973589 0.\n",
      " 0.04200646 0.         0.04023234 0.         0.01117199 0.\n",
      " 0.         0.0185176  0.02148626 0.         0.01067708 0.03430846\n",
      " 0.0925409  0.00232632 0.00847507 0.01184008 0.         0.01498308\n",
      " 0.02681279 0.02111463 0.         0.01594381 0.         0.\n",
      " 0.01104029 0.01191679 0.         0.00653038 0.00778725 0.01914318\n",
      " 0.01490076 0.         0.02123785 0.         0.0048515  0.\n",
      " 0.         0.         0.02520201 0.         0.         0.02154599\n",
      " 0.010742   0.01117199 0.         0.         0.         0.\n",
      " 0.03644198 0.         0.01594021 0.         0.00347664 0.\n",
      " 0.         0.01489599 0.00467966 0.         0.         0.\n",
      " 0.         0.01002024 0.         0.         0.         0.\n",
      " 0.         0.         0.01042282 0.02768371 0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.02087927 0.01923397 0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.01117199 0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.06044928]\n"
     ]
    }
   ],
   "source": [
    "max_depth = max_depth_list[dtree_accuracy_temp.index(max(dtree_accuracy_temp))]\n",
    "dtree_model = DecisionTreeClassifier(max_depth = max_depth).fit(X_train, y_train_20)\n",
    "dtree_predictions = dtree_model.predict(X_test)\n",
    "dtree_accuracy = dtree_model.score(X_test, y_test_20)\n",
    "dtree_cm = confusion_matrix(y_test_20, dtree_predictions)\n",
    "print('Decision tree test accuracy: ', dtree_accuracy)\n",
    "print('\\n')\n",
    "print('Decision tree confusion matrix:\\n', dtree_cm)\n",
    "print('\\n')\n",
    "print('Decision tree important features:\\n', dtree_model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RF test accuracy:  0.9803077941419824\n",
      "\n",
      "\n",
      "RF confusion matrix:\n",
      " [[11848     1]\n",
      " [  237     0]]\n",
      "\n",
      "\n",
      "RF important features:\n",
      " [0.01476372 0.01730409 0.01228524 0.02262188 0.0155764  0.01434223\n",
      " 0.00852005 0.01164569 0.01253041 0.01988306 0.00619136 0.01217992\n",
      " 0.01011856 0.01294712 0.02002974 0.01107686 0.00991385 0.00708399\n",
      " 0.00451068 0.01332685 0.00481612 0.01012377 0.01590135 0.0136299\n",
      " 0.00828704 0.01031262 0.02057122 0.00751954 0.01332083 0.01028505\n",
      " 0.00725769 0.0139904  0.01195051 0.01296028 0.01916044 0.01601004\n",
      " 0.00925624 0.01173724 0.01437907 0.0089828  0.01042648 0.01309711\n",
      " 0.01487259 0.02799399 0.00514555 0.00971687 0.00836871 0.00791131\n",
      " 0.0031272  0.027275   0.0100683  0.00676588 0.00932079 0.01863436\n",
      " 0.00927673 0.00485723 0.01737995 0.00346752 0.01413813 0.00564761\n",
      " 0.00628862 0.00586106 0.00603056 0.00525833 0.         0.00795247\n",
      " 0.01139979 0.00398036 0.         0.00588925 0.00265492 0.01385527\n",
      " 0.02017158 0.01014035 0.0094126  0.00138181 0.00254774 0.00034572\n",
      " 0.00190513 0.00350319 0.00377446 0.00054748 0.00171327 0.00035287\n",
      " 0.00037505 0.00147861 0.00083361 0.00026841 0.00095033 0.\n",
      " 0.00088993 0.         0.00082299 0.00198646 0.00225812 0.00060524\n",
      " 0.00094546 0.00136415 0.00094209 0.00155828 0.         0.00022575\n",
      " 0.0012624  0.00024801 0.         0.00080548 0.00149862 0.0009511\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.02302069 0.00046589 0.00362147 0.0005206\n",
      " 0.00026349 0.01242442 0.00266368 0.00584947 0.0023435  0.00734124\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.07145351]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Random Forest Classifier (tune n_estimators)\n",
    "# Can only handle small n_estimators, or the memory will exceed.\n",
    "\n",
    "rf_model = RandomForestClassifier(n_estimators = 40, n_jobs = -1).fit(X_train, y_train_20)\n",
    "rf_predictions = rf_model.predict(X_test)\n",
    "rf_accuracy = rf_model.score(X_test, y_test_20)\n",
    "rf_cm = confusion_matrix(y_test_20, rf_predictions)\n",
    "print('RF test accuracy: ', rf_accuracy)\n",
    "print('\\n')\n",
    "print('RF confusion matrix:\\n', rf_cm)\n",
    "print('\\n')\n",
    "print('RF important features:\\n', rf_model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline prediction for all the test set:  0\n",
      "Baseline test accuracy:  0.9838656296541453\n"
     ]
    }
   ],
   "source": [
    "# Now try cluster 22\n",
    "# Baseline\n",
    "base_pred = mode(y_train_21)\n",
    "print('Baseline prediction for all the test set: ', base_pred)\n",
    "\n",
    "s = 0\n",
    "for i in range(len(y_test_21)):\n",
    "    if int(y_test_21[i]) == int(base_pred):\n",
    "        s += 1\n",
    "print('Baseline test accuracy: ', s/len(y_test_21))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'max_depth')"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8VvX5//HXOwlhbwKyEUHZMyIuUFotLhBRi6uOWq3fotbRqtWftdRJrbgHWhW1dVcFq6LFAVpQwgh7ywggQyEQdpLr98c50dsYSIDcOXeS6/l43I/7jM8593UYeedzzrk/R2aGc845V9qSoi7AOedcxeQB45xzLi48YJxzzsWFB4xzzrm48IBxzjkXFx4wzjnn4sIDxjnnXFx4wDjnnIsLDxjnnHNxkRJ1AVFq1KiRtWnTJuoynHOuXJk2bdpGM0srrl2lDpg2bdqQkZERdRnOOVeuSFpRknZ+isw551xceMA455yLCw8Y55xzceEB45xzLi48YJxzzsWFB4xzzrm48IBxzjkXFx4wB2D15h28OHk5c9dkk5uXH3U5zjmXkCr1Fy0P1OSl3/L/3pkLQM3UZLq3rEfv1vXp1bo+vVrWp26NKhFX6Jxz0fOAOQBDezWnb9sGTFuxiekrNjFt5SYe/3QpefkGQPvGtYLAaRWEzmFpNZEUcdXOOVe2ZGZR1xCZ9PR0K62hYrbtyiUzazMzVm5m2opNTFuxiewdewCoV6MKvVrVp3fr+vRsVY8eLetRI9Wz3TlXPkmaZmbpxbXzn3KlpGbVFI45rBHHHNYIgPx8Y9nGbUxfsYnpK4PA+XjBegCSk0THprXpHfZwerWqT4v61b2X45yrULwHU4aDXWZv38P0VeFptRWbmLlqM9t35wHQuHZVereu//21nM7N6lA1JbnManPOuZLyHkwCqlujCice0ZgTj2gMQG5ePgvXbf0+cKat3MT7c74BIDUlia7N68Zcy6lH49rVoizfOef2i/dgEmy4/vVbdjJ95Samh9dyZmdlszu8FbpVgxph4NSjV+v6HNGkNinJfqe5c65slbQHE9eAkTQQeAhIBp4xs3sLrW8NPAukAd8BF5pZVrhuJHAawXd1PgKuNTOT9AHQlKD3NQn4nZnlSWoAvAq0AZYD55rZpn3Vl4gBU9iu3DzmrN7y/bWcjBWb2LB1FxDcIt2jVT16t6pPT79F2jlXRiIPGEnJwCLgJCALmAqcZ2bzYtq8DrxrZmMkDQAuNbOLJB0D/A3oFzb9HLjFzD6VVMfMtii4Iv4G8LqZvRIG0ndmdq+km4H6ZnbTvmosDwFTmJmRtWnH9zcOTFuxiflrtxDeIU2X5nV44oLetGxQI9pCnXMVViJcg+kDLDGzZWFBrwCDgXkxbToB14XTnwBvh9MGVANSAQFVgHUAZrYlpvbUsC3hvk8Ip8cAnwL7DJjySBItG9SgZYMaDO7RHPjhFunpKzbx1GfLuPH1TF7+TV+SkvyuNOdcdOJ5Ar85sCpmPitcFisTGBpODwFqS2poZpMJAmdt+BpvZvMLNpI0HlgPbCXoxQA0MbO1AOF749I9nMRVcIv08AHtuf2MTnz59Xc897/lUZflnKvk4hkwRf36XPh83I1Af0kzgP7AaiBXUjugI9CCIJQGSOr3/U7MfkFwHaYqMGC/ipKukJQhKWPDhg37s2m5cHbvFvy8Y2NGfrCAJetzoi7HOVeJxTNgsoCWMfMtgDWxDcxsjZmdZWY9gVvDZdkEvZkpZpZjZjnA+0DfQtvuBMYSnBoDWCepKUD4vr6oosxstJmlm1l6WlrawR5jwpHE3Wd1pUZqMje8numDcTrnIhPPgJkKtJd0qKRUYBhBIHxPUiNJBTXcQnBHGcBKgp5NiqQqBL2b+ZJqxYRICnAqsCDcZixwcTh9MfBOnI4r4TWuXY07z+xK5qrNPPnZ0qjLcc5VUnELGDPLBYYD44H5wGtmNlfSCEmDwmYnAAslLQKaAHeFy98AlgKzCa7TZJrZOKAmMFbSrHD5euDJcJt7gZMkLSa4c+1Ht0RXNqd1a8oZ3Zvx0ITFzF2THXU5zrlKyL9oWc5uU94fm7bt5uQHJ9KwZirvDD/Wh55xzpWKkt6m7F8Dr8Dq10zlvqFdWfDNVh767+Koy3HOVTIeMBXcgA5N+GV6S578bCnTV+5zYAPnnCtVHjCVwG2nd6Rp3erc+FomO8LRm51zLt48YCqB2tWq8LdzurFs4zbu+2BB8Rs451wp8ICpJI45rBGXHNOG5/+3nP8t3Rh1Oc65SsADphK5aWAH2jaqyR9en8XWnXuiLsc5V8F5wFQi1VOTuf/c7qzN3sGd784vfgPnnDsIHjCVTK9W9flt/8N4NWMVHy9YF3U5zrkKzAOmErr25+3pcEhtbnpzNpu27Y66HOdcBeUBUwlVTUnmgXN7sHn7bm4fOzfqcpxzFZQHTCXVqVkdrv1Ze8ZlruHdWWuK38A55/aTB0wl9tv+h9G9ZT1ue3sO67fujLoc51wF4wFTiaUkJ/H3c7qzY3cet7w5m8o88KlzrvR5wFRy7RrX4o8DOzBhwXpen5YVdTnOuQrEA8Zx6TFtOOrQBowYN4+sTdujLsc5V0F4wDiSksT953THzPjjG7PIz/dTZc65g+cB4wBo2aAGt53eif8t/ZYXp6yIuhznXAUQ14CRNFDSQklLJN1cxPrWkiZImiXpU0ktYtaNlDRX0nxJDytQQ9J/JC0I190b0/4SSRskzQxfl8fz2CqiYUe25IQj0rjn/fl8vXFb1OU458q5uAWMpGTgMeAUoBNwnqROhZrdD7xgZt2AEcA94bbHAMcC3YAuwJFA/4JtzKwD0BM4VtIpMft71cx6hK9n4nRoFZYk7hvajaopydzw2kzy/FSZc+4gxLMH0wdYYmbLzGw38AowuFCbTsCEcPqTmPUGVANSgapAFWCdmW03s08Awn1OB1rgSk2TOtUYMbgz01duZvTEZVGX45wrx+IZMM2BVTHzWeGyWJnA0HB6CFBbUkMzm0wQOGvD13gz+9Hwv5LqAWfwQ0ABDA1Pt70hqWXpHUrlMqh7M07pcgijPlrEgm+2RF2Oc66cimfAqIhlhc+53Aj0lzSD4BTYaiBXUjugI0HvpDkwQFK/73cspQAvAw+bWcGv2eOANuHptv8CY4osSrpCUoakjA0bNhz40VVgkrjzzC7UqZ7CDa9lsjs3P+qSnHPlUDwDJguI7UW0AH406JWZrTGzs8ysJ3BruCyboDczxcxyzCwHeB/oG7PpaGCxmT0Ys69vzWxXOPs00LuoosxstJmlm1l6WlrawR1hBdawVlXuHtKVuWu28OjHi6MuxzlXDsUzYKYC7SUdKikVGAaMjW0gqZGkghpuAZ4Np1cS9GxSJFUh6N3MD7e5E6gL/L7QvprGzA4qaO8O3MmdD2ForxY89ulSMldtjroc51w5E7eAMbNcYDgwnuCH/WtmNlfSCEmDwmYnAAslLQKaAHeFy98AlgKzCa7TZJrZuPA25lsJbg6YXuh25GvCW5czgWuAS+J1bJXJ7Wd0onHtqtzweiY79+RFXY5zrhxRZR7gMD093TIyMqIuI+FNWryBi/7xFZcfdyi3nV74TnPnXGUjaZqZpRfXzr/J74p1fPs0Lurbmn988TVfLvs26nKcc+WEB4wrkZtP6UCrBjW48Y1Mtu3Kjboc51w54AHjSqRm1RTuP6c7WZt2cNd7fv+Ec654HjCuxI5s04Arjm/Lv75cyWeL/DtEzrl984Bx++W6kw6nfeNa3PTGLLK374m6HOdcAvOAcfulWpVkHji3BxtzdnHHuLlRl+OcS2AeMG6/dW1Rl+ED2vHWjNV8MGdt1OU45xKUB4w7IL87sR1dmtfh1rfmsDFnV/EbOOcqHQ8Yd0CqJCfxwLk92Lorl1vfmk1l/sKuc65oHjDugB3epDY3nnw44+eu4+2Zq6MuxzmXYDxg3EH59XFtSW9dn9vfmcva7B1Rl+OcSyAeMO6gJCeJ+8/pTm6ecdObfqrMOfcDDxh30No0qsmfTuvIxEUb+NdXK6MuxzmXIDxgXKm48KhWHN++EXf9Zz4rv90edTnOuQTgAeNKhSTuG9qN5CRx4+uZ5OX7qTLnKjsPGFdqmtWrzh1ndOar5d/x3BdfR12Ocy5iHjCuVJ3VqzkndWrCyPELWbxua9TlOOciFNeAkTRQ0kJJSyTdXMT61pImSJol6dPwkcgF60aGj0CeL+lhBWpI+o+kBeG6e2PaV5X0avhZX0pqE89jc0WTxN1DulKrago3vJ7Jnrz8qEtyzkUkbgEjKRl4DDgF6AScJ6nw83bvB14ws27ACOCecNtjgGOBbkAX4Eigf8E2ZtYB6AkcK+mUcPmvgU1m1g4YBdwXr2Nz+5ZWuyp3ntmFWVnZPPHp0qjLcc5FJJ49mD7AEjNbZma7gVeAwYXadAImhNOfxKw3oBqQClQFqgDrzGy7mX0CEO5zOlDQ6xkMjAmn3wB+JkmlflSuRE7t2pTBPZrx8ITFzFmdHXU5zrkIxDNgmgOrYuazwmWxMoGh4fQQoLakhmY2mSBw1oav8Wb2o8coSqoHnMEPAfX955lZLpANNCy1o3H77S+DOtOgZirXvzaTXbl5UZfjnCtj8QyYonoPhe9dvRHoL2kGwSmw1UCupHZAR4LeSXNggKR+3+9YSgFeBh42s2X78XlIukJShqSMDRv8qYzxVK9GKved3Y1F63IY9dHiqMtxzpWxeAZMFtAyZr4FsCa2gZmtMbOzzKwncGu4LJugNzPFzHLMLAd4H+gbs+loYLGZPVjU54UBVBf4rnBRZjbazNLNLD0tLe1gj9EV48QjGnNen5aMnriUZyYtI9+/H+NcpRHPgJkKtJd0qKRUYBgwNraBpEaSCmq4BXg2nF5J0LNJkVSFoHczP9zmToLw+H2hzxsLXBxOnw18bD4wVkK47bRODOjQmDv/M59hT0/xb/o7V0nELWDC6yDDgfEE4fCamc2VNELSoLDZCcBCSYuAJsBd4fI3gKXAbILrNJlmNi68jflWgpsDpkuaKenycJt/AA0lLQGuB35yW7SLRs2qKTz9q3T+dnY35q/ZwsCHJvLSlBU+MKZzFZwq83/y9PR0y8jIiLqMSmXN5h3c9OYsJi3eyPHtG3Hf0G40q1c96rKcc/tB0jQzSy+unX+T35WpZvWq88JlfbjzzC5MW7GJX4yayOsZq7w341wF5AHjypwkLuzbmg+u7UfHpnX4wxuz+M0LGazfujPq0pxzpcgDxkWmVcMavHJFX247rSOTFm/k5FETGZe5pvgNnXPlggeMi1RSkrj8+Lb855rjad2wJle/PIPf/XM6323bHXVpzrmD5AHjEkK7xrV487dH84dfHMGH877h5FGf8eHcb6Iuyzl3EDxgXMJISU7idye2Y+zw42hcuxpXvDiN61+bSfaOPVGX5pw7AB4wLuF0bFqHt393LNcMaMc7M9fwi1ET+WyRD+vjXHnjAeMSUmpKEteffARv/d8x1KqWwsXPfsUt/55Nzq7cqEtzzpWQB4xLaN1a1OPdq4/jyn5teWXqSgY+OJHJS7+NuiznXAl4wLiEV61KMrec2pHXrzyalCRx3tNT+Mu4uezY7Y8AcC6RecC4ciO9TQPeu/Z4Lj66Nc99sZxTH57EtBWboi7LObcXJQoYSW9KOi1m5GPnIlEjNYW/DO7Cvy4/it25+Zzz5P+49/0F/kAz5xJQSQPjCeB8YLGkeyV1iGNNzhXrmHaN+OD3x3Nuekue/Gwpgx75wh/N7FyCKVHAmNl/zewCoBewHPhI0v8kXRo+r8W5Mle7WhXuHdqN5y49kk3bd3PmY1/w4H8XsScvP+rSnHPsxzUYSQ2BS4DLgRnAQwSB81FcKnOuhE48ojEfXteP07s15cH/LmbI41+w8JutUZflXKVX0msw/wYmATWAM8xskJm9amZXA7XiWaBzJVGvRioPDuvJkxf2Zu3mnZzxyOc88elS8vwRzc5FpkQPHJM0wMw+LoN6ypQ/cKxi+jZnF7e9PYf353xDr1b1uP+c7rRN89+DnCstJX3gWEoJ99dR0nQz2xzuvD5wnpk9XkwRAwlOpSUDz5jZvYXWtwaeBdKA74ALzSwrXDcSOI2gl/URcK2ZmaS7gF8B9c2sVsy+LgH+BqwOFz1qZs+U8PhcBdKwVlUev6AXYzPXcPs7czn14UncNLADFx/dhqQkRV1ekXbn5rNuy07WbN7B2uydrN+6k9wE7H2lJidxdu8W1KuRGnUprhwoaQ9mppn1KLRshpn13Mc2ycAi4CQgC5hKEErzYtq8DrxrZmMkDQAuNbOLJB1DEBb9wqafA7eY2aeS+gIrgMVFBEy6mQ0vyYGD92Aqg3VbdnLzm7P4ZOEG+rZtwN/O7k7LBjXKtIa8fGP91p2s2byTtdk7WLt5J2vC97XZO1iTvZONObsoLw/1PLJNff55eV9SU/xbC5VVafdgkiTJwjQKw6O4X2H6AEvMbFm4zSvAYGBeTJtOwHXh9CfA2+G0AdXCzxBQBVgHYGZTwv2VsHRXmTWpU41nLzmS1zOyGPHuPAY+OJFbT+vEeX1alsq/ofx8Y+O2XT+ERUxorN28g2+yd7Ju666fXAuqmZpM03rVaVq3GkccUpumdavTrF6179+b1KlGleTE+wE+fu43XPvKTEa8O5c7z+wadTkuwZU0YMYDr0l6kuCH/2+BD4rZpjmwKmY+CziqUJtMYCjBabQhQG1JDc1ssqRPgLUEAfOomc0vQZ1DJfUj6DldZ2aritvAVXySOPfIlhzTriE3vTmLP701mw/mfsN9Q7vStG71vW5nZmzevucnvY21m8P37B2sy97F7kK3RVdNSaJp3SAs+h7WkGZ1q9O0XrXv35vWrU6dainl8pekwT2aM2/tFp76bBmdm9XlvD6toi7JJbCSBsxNwJXAVQQ/8D8Eiru+UdT/nsInAW4EHg1Pb00kuH6SK6kd0BFoEbb7SFI/M5u4j88bB7xsZrsk/RYYAwz4SVHSFcAVAK1a+X+OyqRF/Rq8eNlR/PPLFdz93gJOHjWRW0/tSJO61X7SA1kbBsjOPT8OjyrJokmdICx6tar/o55HECrVaFAztVyGR0n98RcdmLdmC7e/M4fDm9Sid+sGUZfkElSJrsEc0I6lo4E7zOwX4fwtAGZ2z17a1wIWmFkLSX8AqpnZX8N1twM7zWxkTPuc2GswhfaVDHxnZnX3VaNfg6m8lm/cxo2vZ5IRM5ZZkoJTak3rVgtOX9UJ3pvV/eG9Ua2qCXujQFnavH03gx/7gu2783j36uNoUqda1CW5MlSq12AktQfuIbhm8v2/JDNru4/NpgLtJR1K0DMZRjDcTOx+GxEEQT5wC8EdZQArgd9IuoegJ9QfeLCYGpua2dpwdhBQklNqrpJq06gmr155NJOXfkv11CSa1q1O49pVSUnA6x6JqF6NVEZflM6Qx7/gyhen8eqVfamakhx1WS7BlPR/03ME45HlAicCLwAv7msDM8sFhhNcv5kPvGZmcyWNkDQobHYCsFDSIqAJcFe4/A1gKTCb4DpNppmNg+D2ZUlZQA1JWZLuCLe5RtJcSZnANQSjDji3V8lJ4rj2jejdugHN6lX3cNlPRxxSmwfO7c7MVZv5f2/PIV5nQ1z5VdLblKeZWW9Js82sa7hskpkdH/cK48hPkTl38P7+4UIe+XgJfx3cmYuObhN1Oa4MlPZtyjvDofoXSxpOcMqr8cEU6JyrGK77+eHMW7OFv4ybx+FNanNU24ZRl+QSREnPCfyeYByya4DewIXAxfEqyjlXfiQliVHDetCqYQ3+75/TWb15R9QluQRRbMCEd2Sda2Y5ZpZlZpea2dCCLzw651ydalUYfVE6u3LzufLFDHbu8QfAuRIEjJnlAb1VkW/sd84dtHaNa/HgL3swZ/UWbvn3bL/o70p8DWYG8E44dti2goVm9u+4VOWcK5d+3qkJ1590OA98tIjOzepw+fH7+iaDq+hKGjANgG/58TfjDfCAcc79yPAT2zF3TTZ3vzefDofU4bj2jaIuyUUkbt/kLw/8NmXn4iNnVy5nPf4F67fuYtzw48p8BGsXXyW9TbmkT7R8TtKzhV8HX6ZzriKqVTWF0Relk59v/OaFDLbvzo26JBeBkt6m/C7wn/A1AagD5MSrKOdc+demUU0eOb8Xi9Zt5Q9vzPKL/pVQia7BmNmbsfOSXgb+G5eKnHMVRv/D0/jjwA7c+/4CujSry1UnHBZ1Sa4MHejgS+0BH+veOVesK/u15fRuTRk5fgGfLlwfdTmuDJX0GsxWSVsKXgTPXrkpvqU55yoCSYw8uxsdDqnDNS/P4OuN24rfyFUIJQoYM6ttZnViXocXPm3mnHN7UyM1hdEX9SY5SVzxQgY5u/yif2VQ0h7MEEl1Y+brSTozfmU55yqalg1q8Oj5vVi2cRvXvzqT/Hy/6F/RlfQazJ/NLLtgxsw2A3+OT0nOuYrq2HaN+NOpHflw3joe/WRJ1OW4OCtpwBTVrqSjADjn3PcuO7YNZ/VszgMfLeKjeeuiLsfFUUkDJkPSA5IOk9RW0ihgWjwLc85VTJK4+6yudG1el+tencmS9f6VuoqqpAFzNbAbeBV4DdgB/K64jSQNlLRQ0hJJNxexvrWkCZJmSfpUUouYdSPDRyDPl/RwwWjOku6StEpSTqF9VZX0avhZX0pqU8Jjc86VsWpVknnqot5UTUniihcy2LJzT9QluTgo6V1k28zsZjNLD19/MrN93msYPkfmMeAUoBNwnqROhZrdD7xgZt2AEcA94bbHAMcC3YAuwJFA/3CbcUCfIj7y18AmM2sHjALuK8mxOeei0axedR6/oBcrv9vO71/xi/4VUUnvIvtIUr2Y+fqSxhezWR9giZktM7PdwCvA4EJtOhEMPQPwScx6A6oBqUBVoAqwDsDMppjZ2iI+bzAwJpx+A/iZP8PGucR2VNuG/PmMTny8YD2j/rso6nJcKSvpKbJG4Z1jAJjZJqBxMds0B1bFzGeFy2JlAkPD6SFAbUkNzWwyQeCsDV/jzWx+ST/PzHKBbMAfDu5cgruwb2t+md6SRz5ewvuzi/rd0ZVXJQ2YfEnfDw0TXt8orj9bVO+h8DY3Av0lzSA4BbYayJXUDugItCAIjgGS+pXC5yHpCkkZkjI2bNhQzC6dc/EmiRFndqZnq3rc8HomC7/ZGnVJrpSUNGBuBT6X9KKkF4HPgFuK2SYLaBkz3wJYE9vAzNaY2Vlm1jP8DMLv2wwBpphZjpnlAO8DfUv6eZJSgLrAd4UbmdnogmtJaWlpxezSOVcWqqYk8+SFvalZNYXfvJDB5u27oy7JlYKSXuT/AEgHFhLcSXYDwZ1k+zIVaC/pUEmpwDBgbGwDSY0kFdRwC1DwjJmVBD2bFElVCHo3xZ0iGwtcHE6fDXxsPj64c+VGkzrVePLC3qzN3sHVL88gzy/6l3slvch/OcHF+BvC14vAHfvaJrwOMhwYTxAOr5nZXEkjJA0Km50ALJS0CGgC3BUufwNYCswmuE6TaWbjwlpGSsoCakjKklRQxz+AhpKWANcDP7kt2jmX2Hq3rs9fB3dh0uKNjPxgQdTluINUokcmS5pNcKvwFDPrIakD8Bcz+2W8C4wnf2Syc4nptrdn89KUlTw0rAeDexS+N8hFrVQfmQzsNLOd4Y6rmtkC4IiDKdA55/bm9tM7c2Sb+tz05izmrM4ufgOXkEoaMFnh92DeBj6S9A6FLtg751xpSU1J4vELelOveipXvjiNb3N2RV2SOwAlvcg/xMw2m9kdwP8juN7hw/U75+ImrXZVnrqoNxtydjH8XzPYk5cfdUluP+33I5PN7DMzGxt+O9855+Kme8t63DOkK5OXfcvd7xV3I6lLND7kvnMuoQ3t3YI5a7J57ovldG5Wl7N7tyh+I5cQ9rsH45xzZe1Pp3bk6LYN+dNbs8lctbn4DVxC8IBxziW8KslJPHp+T9JqVeXKF6exYatf9C8PPGCcc+VCw1rBRf/NO3bzf/+cxu5cv+if6DxgnHPlRpfmdblvaDemLt/EiHfnRl2OK4Zf5HfOlSuDezRn3potPDVxGZ2b1eW8Pq2K38hFwnswzrly548DO3B8+0bc/s4cpq34yaDpLkF4wDjnyp3kJPHIeT1pVq86v31pOuu27Iy6JFcEDxjnXLlUr0Yqoy9KZ9uuXK58cRo79+RFXZIrxAPGOVduHXFIbR44tzszV23m3Kcms3idPw0zkXjAOOfKtYFdmvLEBb1Y9d12Tnvkc0ZPXOoPK0sQHjDOuXLvlK5N+fC6/pxweBp3v7eAXz41meUbt0VdVqXnAeOcqxAKRl9+4NzuLFy3lVMemsQLk5eT772ZyMQ1YCQNlLRQ0hJJP3mEsaTWkiZImiXpU0ktYtaNlDRX0nxJD0tSuLy3pNnhPmOX3yFptaSZ4evUeB6bcy7xSOKsXi346Lr+9Dm0Abe/M5cL//ElWZu2R11apRS3gJGUDDwGnAJ0As6T1KlQs/uBF8ysGzACuCfc9hjgWKAb0IXgcc39w22eAK4A2oevgTH7G2VmPcLXe3E5MOdcwjukbjWev/RI7jmrK5mrNjPwwUm8OnUlJXlEvCs98ezB9AGWmNmy8NkxrwCDC7XpBEwIpz+JWW9ANSAVqApUAdZJagrUMbPJFvxLeQF/8JlzrgiSOK9PKz74fT+6NK/DTW/O5rLnp/p3ZspQPAOmObAqZj4rXBYrExgaTg8BaktqaGaTCQJnbfgab2bzw+2z9rHP4eHptmcl1S+9Q3HOlVctG9TgX5f35c9ndGLysm85edRE3pm52nszZSCeAaMilhX+G70R6C9pBsEpsNVArqR2QEegBUGADJDUr5h9PgEcBvQgCKW/F1mUdIWkDEkZGzZs2M9Dcs6VR0lJ4tJjD+W9a47nsLSaXPvKTK56aTobc3zY/3iKZ8BkAS1j5lsAa2IbmNkaMzvLzHoCt4bLsgl6M1PMLMfMcoD3gb7hPlsUtU8zW2dmeWaWDzxNcIruJ8xstJmlm1l6WlpaaRync66caJtWi9d/eww3n9KBjxes5xejJvLBnLVRl1VhxTNgpgLtJR0STRH8AAARiklEQVQqKRUYBoyNbSCpkaSCGm4Bng2nVxL0bFIkVSHo3cw3s7XAVkl9w7vHfgW8E+6racyuhwBz4nVgzrnyKzlJ/Lb/YYy7+jia1qvGb1+azu9fmUH29j1Rl1bhxC1gzCwXGA6MB+YDr5nZXEkjJA0Km50ALJS0CGgC3BUufwNYCswmuE6TaWbjwnVXAc8AS8I274fLR4a3L88CTgSui9exOefKvyMOqc1b/3cs1/38cN6dtZaTRn3GJwvWR11WhaLKfKErPT3dMjIyoi7DORexOauzueG1TBau28ov01ty2+kdqV2tStRlJSxJ08wsvbh2/k1+51yl16V5XcZefSxXnXAYr09bxcAHJ/HFko1Rl1XuecA45xxQNSWZmwZ24I2rjqFqShIXPPMlf35nDtt350ZdWrnlAeOcczF6tarPf645nsuOPZQxk1dw6kOTyFjuT808EB4wzjlXSPXUZG4/oxOvXNGXPDPOeWoyd7833x9qtp88YJxzbi/6tm3I+9f247w+rRg9cRmnP/I5s7I2R11WueEB45xz+1Cragp3D+nKmMv6kLMzlyGP/48HPlzI7tz8qEtLeB4wzjlXAv0PT2P8df04s0dzHv54CWc+9gXz126JuqyE5gHjnHMlVLd6Ff5+bndGX9Sb9Vt3MujRz3nskyXk5nlvpigeMM45t59O7nwIH17Xn5M7H8Lfxi9k6JOTWbI+J+qyEo4HjHPOHYAGNVN57PxePHJeT1Z8u43THp7EM5OW+SOaY3jAOOfcQTijezM+vK4fx7dP487/zGfY6Cms+HZb1GUlBA8Y55w7SI1rV+PpX/Xm/nO6M3/tFk55aBIvTVlR6R9q5gHjnHOlQBJn927B+Ov60bt1fW57ew4X/eOrSv2IZg8Y55wrRc3qVeeFy/pw15AuTF+5iWGjp1TakPGAcc65UiaJC45qzYu/7sP6LTs57+kprN9a+ULGA8Y55+Kkd+sGPH9ZH77J3skFT3/JxpxdUZdUpjxgnHMujo5s04BnLzmSVZu2c8HTX/Ldtt1Rl1Rm4howkgZKWihpiaSbi1jfWtIESbMkfSqpRcy6kZLmSpov6WFJCpf3Dh+NvKTQ8gaSPpK0OHyvH89jc865kurbtiHPXnwky7/dxgXPfMmmShIycQsYScnAY8ApQCfgPEmdCjW7H3jBzLoBI4B7wm2PAY4FugFdgCOB/uE2TwBXAO3D18Bw+c3ABDNrD0wI551zLiEc064RT/8qnaUbcrjwH1+SvX1P1CXFXTx7MH2AJWa2zMx2A68Agwu16UQQBgCfxKw3oBqQClQFqgDrJDUF6pjZZAtuMH8BODPcZjAwJpweE7PcOecSQr/D03jqot4sXpfDRc9+SfaOih0y8QyY5sCqmPmscFmsTGBoOD0EqC2poZlNJgicteFrvJnND7fP2ss+m5jZWoDwvXEpHotzzpWKE49ozBMX9mL+2i1c/OxXbN1ZcUMmngGjIpYV/lrrjUB/STMIToGtBnIltQM6Ai0IAmSApH4l3Oe+i5KukJQhKWPDhg37s6lzzpWKn3VswmPn92LO6mwueW4qObtyoy4pLuIZMFlAy5j5FsCa2AZmtsbMzjKznsCt4bJsgt7MFDPLMbMc4H2gb7jPFnvZZ8EpNML39UUVZWajzSzdzNLT0tIO9hidc+6AnNz5EB49vyczV23m0ue+YlsFDJl4BsxUoL2kQyWlAsOAsbENJDWSVFDDLcCz4fRKgp5NiqQqBL2b+eGpr62S+oZ3j/0KeCfcZixwcTh9ccxy55xLSAO7NOXhYT2ZvnIzlz0/le27K1bIxC1gzCwXGA6MB+YDr5nZXEkjJA0Km50ALJS0CGgC3BUufwNYCswmuE6TaWbjwnVXAc8AS8I274fL7wVOkrQYOCmcd865hHZat6Y8cG53pi7/jsvHZLBjd17UJZUaVebRPtPT0y0jIyPqMpxzjn9Pz+KG1zM5LryduVqV5KhL2itJ08wsvbh2/k1+55xLAGf1asHIod34fMlGrnxxGrtyy39PxgPGOecSxDnpLblnSFc+W7SBq16azu7c/KhLOigeMM45l0CG9WnFXUO68PGC9fzuX9PZk1d+Q8YDxjnnEswFR7VmxODOfDRvHde8PKPchowHjHPOJaBfHd2G20/vxPtzvuH3r84ktxyGTErUBTjnnCvaZccdSl6+cdd780mWGPXLHiQnFTWgSWLygHHOuQT2m35tyc037vtgASlJ4m/ndC83IeMB45xzCe6qEw4jLz+f+z9cRFKSGDm0G0nlIGQ8YJxzrhwYPqA9ufnGg/9dTEqSuHtI14QPGQ8Y55wrJ679WXvy8o1HPl5CcpK488wuhA/1TUgeMM45V05I4vqTDmdPnvHkZ0tJSRJ3DOqcsCHjAeOcc+WIJG4aeAR5+fk8PelrkpLE7ad3SsiQ8YBxzrlyRhJ/OrUjufnGc18sJyUpmE+0kPGAcc65ckgKei55+cbTk74mJTmJP/7iiIQKGQ8Y55wrpyTxl0Gdycs3nvh0KVWSxPUnHxF1Wd/zgHHOuXJMEn8d3IW8fOPhj5eQnJTEtT9vH3VZgAeMc86Ve0nh92Jy841R/11EclLwvZmoxXWwS0kDJS2UtETSzUWsby1pgqRZkj6V1CJcfqKkmTGvnZLODNcNkDRd0hxJYySlhMtPkJQds83t8Tw255xLJElJ4r6h3RjSszn3f7iIJz5dGnVJ8evBSEoGHgNOArKAqZLGmtm8mGb3Ay+Y2RhJA4B7gIvM7BOgR7ifBsAS4ENJScAY4GdmtkjSCOBi4B/h/iaZ2enxOibnnEtkyUni/nO6kxczdtlv+rWNrJ549mD6AEvMbJmZ7QZeAQYXatMJmBBOf1LEeoCzgffNbDvQENhlZovCdR8BQ0u9cuecK6eSk8QD53bntK5Nueu9+Tz7+deR1RLPgGkOrIqZzwqXxcrkh4AYAtSW1LBQm2HAy+H0RqCKpPRw/mygZUzboyVlSnpfUueDPQDnnCuPUpKTeHBYDwZ2PoQR787jhcnLI6kjngFT1M3YVmj+RqC/pBlAf2A1kPv9DqSmQFdgPICZGUHgjJL0FbA1pv10oLWZdQceAd4usijpCkkZkjI2bNhwoMfmnHMJrUpyEg+f15OTOjXh9nfm8s8vV5R5DfEMmCx+3LtoAayJbWBma8zsLDPrCdwaLsuOaXIu8JaZ7YnZZrKZHW9mfYCJwOJw+RYzywmn3yPo6TQqXJSZjTazdDNLT0tLK5UDdc65RJSaksSj5/dkQIfG3PrWHF6durJMPz+eATMVaC/pUEmpBD2PsbENJDUKL9wD3AI8W2gf5/HD6bGCbRqH71WBm4Anw/lDFH6FVVIfgmP7tlSPyDnnypmqKck8fkEv+h+exs3/ns3rGauK36iUxC1gzCwXGE5wems+8JqZzZU0QtKgsNkJwEJJi4AmwF0F20tqQ9AD+qzQrv8gaT4wCxhnZh+Hy88G5kjKBB4GhoWn1JxzrlKrViWZpy7qzXHtGvHHN2fx1oysMvlcVeafwenp6ZaRkRF1Gc45VyZ27M7j12OmMmXZt4z6ZQ8G9yh831XJSJpmZunFtYvrFy2dc84ljuqpyTxzcTp92zYkNy/+nQsfKsY55yqRGqkpvPTro8rkccveg3HOuUqmLMIFPGCcc87FiQeMc865uPCAcc45FxceMM455+LCA8Y551xceMA455yLCw8Y55xzcVGph4qRtAE40DGsGxE8nybRJGpdkLi1eV37x+vaPxWxrtZmVuxw9JU6YA6GpIySjMVT1hK1Lkjc2ryu/eN17Z/KXJefInPOORcXHjDOOefiwgPmwI2OuoC9SNS6IHFr87r2j9e1fyptXX4NxjnnXFx4D8Y551xceMAcAEnXSZoraY6klyVVi7omAEnXhjXNlfT7COt4VtJ6SXNiljWQ9JGkxeF7/QSp65zwzytfUiR3+uylrr9JWiBplqS3JNVLkLr+GtY0U9KHkpqVdV17qy1m3Y2STFKjqGuSdIek1eGf10xJp5ZlTfuqLVx+taSF4f+BkaX9uR4w+0lSc+AaIN3MugDJwLBoqwJJXYDfAH2A7sDpktpHVM7zwMBCy24GJphZe2BCOF/Wnuendc0BzgImlnk1P3ien9b1EdDFzLoBi4Bbyrooiq7rb2bWzcx6AO8Ct5d5VYHn+WltSGoJnASsLOuC2EtNwCgz6xG+3ivjmgo8T6HaJJ0IDAa6mVln4P7S/lAPmAOTAlSXlALUANZEXA9AR2CKmW03s1zgM2BIFIWY2UTgu0KLBwNjwukxwJllWhRF12Vm881sYVnXUqiGour6MPx7BJgCtEiQurbEzNYEIrmIu5d/YwCjgD8SQV37qClye6ntKuBeM9sVtllf2p/rAbOfzGw1QdKvBNYC2Wb2YbRVAcFv4v0kNZRUAzgVaBlxTbGamNlagPC9ccT1lCeXAe9HXUQBSXdJWgVcQHQ9mJ+QNAhYbWaZUddSyPDwtOKzUZwa3ofDgeMlfSnpM0lHlvYHeMDsp/AfyGDgUKAZUFPShdFWFfwmDtxHcGrlAyATyN3nRi7hSbqV4O/xn1HXUsDMbjWzlgQ1DY+6HoDwl6pbSaDACz0BHAb0IPiF9O/RlvMjKUB9oC/wB+A1SaX6LGUPmP33c+BrM9tgZnuAfwPHRFwTAGb2DzPrZWb9CLrDi6OuKcY6SU0BwvdS745XNJIuBk4HLrDE/D7Bv4ChURcROozgl75MScsJTilOl3RIlEWZ2TozyzOzfOBpgmukiSIL+LcFvgLyCcYnKzUeMPtvJdBXUo0w7X8GzI+4JgAkNQ7fWxFcuH452op+ZCxwcTh9MfBOhLUkPEkDgZuAQWa2Pep6ChS6cWQQsCCqWmKZ2Wwza2xmbcysDcEPz15m9k2UdRX8UhUaQnAqO1G8DQwAkHQ4kEppD8ppZv7azxfwF4L/WHOAF4GqUdcU1jUJmEdweuxnEdbxMsHpgD0E/9F/DTQkuHtscfjeIEHqGhJO7wLWAeMTpK4lwCpgZvh6MkHqejP8dz8LGAc0T5R/Y4XWLwcaRV1T+PNhdvjnNRZomih/XgSB8lL49zkdGFDan+vf5HfOORcXforMOedcXHjAOOeciwsPGOecc3HhAeOccy4uPGCcc87FhQeMc865uPCAca4ckLT8QIefl3RJ7LD6B7Mv5/aHB4xzFd8lBOPmOVemPGCc2w+S2oQPAnsmfLjbPyX9XNIX4cPU+oSv/0maEb4fEW57vaRnw+mu4fY19vI5DcMHes2Q9BSgmHUXSvoqfIDVU5KSw+U5kv4uabqkCZLSJJ0NpAP/DNtXD3dzddhutqQO8fwzc5WXB4xz+68d8BDQDegAnA8cB9wI/IlgGKF+ZtaTYHTfu8PtHgTaSRoCPAdcaXsfZ+zPwOfhPsYCrQAkdQR+CRxrwUO/8giGzYfg+SzTzawXwfOA/mxmbwAZBANm9jCzHWHbjWG7J8K6nSt1KVEX4Fw59LWZzQaQNJfgSZ0maTbQBqgLjAkHhjSgCoCZ5Uu6hGBcqqfM7It9fEY/ggFLMbP/SNoULv8Z0BuYGo6sXp0fRqbOB14Np18iGOl7bwrWTSv4HOdKmweMc/tvV8x0fsx8PsH/qb8Cn5jZEEltgE9j2rcHcijZNZGiBgoUMMbMSvII5X0NNFhQcx7+c8DFiZ8ic6701QVWh9OXFCyUVJfg1Fo/oGF4fWRvJhKe+pJ0CsGDoSAYifrsmEczNJDUOlyXBBTs83zg83B6K1D7II7HuQPiAeNc6RsJ3CPpCyA5Zvko4HEzW0QwXPq9BUFRhL8QPAJ7OnAywXOIMLN5wG3Ah5JmETzBtOCZI9uAzpKmETznY0S4/HngyUIX+Z2LOx+u37kKQlKOmdWKug7nCngPxjnnXFx4D8a5CEm6FLi20OIvzOx3UdTjXGnygHHOORcXforMOedcXHjAOOeciwsPGOecc3HhAeOccy4uPGCcc87Fxf8H/gvPyj9aA5kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Decision tree classification (tune max_depth)\n",
    "max_depth_list = [8,9,10,11,12,13,14,15,16]\n",
    "dtree_accuracy_temp = []\n",
    "for max_depth in max_depth_list:\n",
    "    dtree_model_temp = DecisionTreeClassifier(max_depth = max_depth).fit(X_train, y_train_21)\n",
    "    dtree_accuracy_temp.append(dtree_model_temp.score(X_test, y_test_21))\n",
    "    \n",
    "plt.plot(max_depth_list, dtree_accuracy_temp)\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('max_depth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision tree test accuracy:  0.9830382260466656\n",
      "\n",
      "\n",
      "Decision tree confusion matrix:\n",
      " [[11880    11]\n",
      " [  194     1]]\n",
      "\n",
      "\n",
      "Decision tree important features:\n",
      " [0.04823442 0.00747503 0.01396568 0.04874773 0.         0.\n",
      " 0.05129121 0.02053901 0.008418   0.         0.01396568 0.06590143\n",
      " 0.         0.         0.         0.         0.02128103 0.04344877\n",
      " 0.04344877 0.02034334 0.01044667 0.         0.         0.\n",
      " 0.         0.0167747  0.0186209  0.01163806 0.         0.\n",
      " 0.         0.0289714  0.         0.         0.02696129 0.\n",
      " 0.         0.         0.         0.         0.         0.07084154\n",
      " 0.00730764 0.01392654 0.01713112 0.         0.         0.01396568\n",
      " 0.         0.0298697  0.         0.         0.         0.02570165\n",
      " 0.         0.         0.         0.01067878 0.01023519 0.\n",
      " 0.         0.         0.         0.         0.         0.05386552\n",
      " 0.         0.         0.         0.         0.00145892 0.01396568\n",
      " 0.         0.0186209  0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.02456136 0.\n",
      " 0.         0.03132379 0.         0.00497833 0.         0.\n",
      " 0.02502318 0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.01592534\n",
      " 0.         0.         0.         0.01098035 0.         0.01162316\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.00770596 0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.05983656]\n"
     ]
    }
   ],
   "source": [
    "max_depth = max_depth_list[dtree_accuracy_temp.index(max(dtree_accuracy_temp))]\n",
    "dtree_model = DecisionTreeClassifier(max_depth = max_depth).fit(X_train, y_train_21)\n",
    "dtree_predictions = dtree_model.predict(X_test)\n",
    "dtree_accuracy = dtree_model.score(X_test, y_test_21)\n",
    "dtree_cm = confusion_matrix(y_test_21, dtree_predictions)\n",
    "print('Decision tree test accuracy: ', dtree_accuracy)\n",
    "print('\\n')\n",
    "print('Decision tree confusion matrix:\\n', dtree_cm)\n",
    "print('\\n')\n",
    "print('Decision tree important features:\\n', dtree_model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RF test accuracy:  0.9838656296541453\n",
      "\n",
      "\n",
      "RF confusion matrix:\n",
      " [[11891     0]\n",
      " [  195     0]]\n",
      "\n",
      "\n",
      "RF important features:\n",
      " [1.63099712e-02 1.77632756e-02 1.64495295e-02 2.92190463e-02\n",
      " 1.77296596e-02 1.68813309e-02 9.95537291e-03 1.16428182e-02\n",
      " 1.18099958e-02 1.75494945e-02 6.53338590e-03 1.69829945e-02\n",
      " 1.09809472e-02 1.25912190e-02 1.91107884e-02 8.26874402e-03\n",
      " 1.09475656e-02 8.83813829e-03 7.24766277e-03 1.26147455e-02\n",
      " 6.43776888e-03 1.04629652e-02 1.95775112e-02 1.07731820e-02\n",
      " 8.24121503e-03 9.26713240e-03 2.24997095e-02 8.98751982e-03\n",
      " 1.44841900e-02 7.66553246e-03 6.21440709e-03 9.99275232e-03\n",
      " 8.31850129e-03 1.41788486e-02 1.81904523e-02 1.54329521e-02\n",
      " 9.62908362e-03 1.11381096e-02 1.25547710e-02 7.08041331e-03\n",
      " 9.93461923e-03 1.72287423e-02 1.46146765e-02 2.34772910e-02\n",
      " 5.61926161e-03 1.49234172e-02 5.64306229e-03 6.27449895e-03\n",
      " 1.90199247e-03 2.77177314e-02 1.11134940e-02 6.89508833e-03\n",
      " 1.08805073e-02 1.58437354e-02 8.73187899e-03 4.85846529e-03\n",
      " 1.44727943e-02 5.50015985e-03 1.11161543e-02 5.30247536e-03\n",
      " 4.83222409e-03 4.10548464e-03 3.62593241e-03 3.57464357e-03\n",
      " 0.00000000e+00 8.07293337e-03 1.26474550e-02 4.02764487e-03\n",
      " 0.00000000e+00 4.86858092e-03 2.62663930e-03 1.45335573e-02\n",
      " 2.03284491e-02 1.07833619e-02 6.98352387e-03 6.56074486e-04\n",
      " 1.38860413e-03 1.67123335e-04 2.01002109e-03 2.36449893e-03\n",
      " 2.66408439e-03 9.32140495e-04 5.30305113e-04 6.97471814e-04\n",
      " 1.01322112e-04 7.18928481e-04 1.37200065e-03 1.16228285e-03\n",
      " 8.70224765e-05 0.00000000e+00 9.85570937e-04 0.00000000e+00\n",
      " 1.13185670e-03 1.18850565e-03 2.01972241e-03 5.17166161e-04\n",
      " 4.07909164e-04 1.35212623e-03 6.33055942e-04 1.48284773e-03\n",
      " 0.00000000e+00 2.29564530e-03 1.01296164e-03 4.90091920e-04\n",
      " 0.00000000e+00 1.42803099e-03 1.04483035e-03 1.36667514e-03\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 2.20365493e-02 3.84172876e-04\n",
      " 2.44014561e-03 3.06845210e-04 9.70037282e-04 1.26116200e-02\n",
      " 3.24779938e-03 8.21513646e-03 1.65707159e-03 5.55048588e-03\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00\n",
      " 0.00000000e+00 0.00000000e+00 7.67570799e-02]\n"
     ]
    }
   ],
   "source": [
    "# Random Forest Classifier (tune n_estimators)\n",
    "# Can only handle small n_estimators, or the memory will exceed.\n",
    "\n",
    "rf_model = RandomForestClassifier(n_estimators = 40, n_jobs = -1).fit(X_train, y_train_21)\n",
    "rf_predictions = rf_model.predict(X_test)\n",
    "rf_accuracy = rf_model.score(X_test, y_test_21)\n",
    "rf_cm = confusion_matrix(y_test_21, rf_predictions)\n",
    "print('RF test accuracy: ', rf_accuracy)\n",
    "print('\\n')\n",
    "print('RF confusion matrix:\\n', rf_cm)\n",
    "print('\\n')\n",
    "print('RF important features:\\n', rf_model.feature_importances_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Appendix: Supervised learning methods that exceed our computation limits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "# SVM classification (tune C)\n",
    "svm_model_linear = SVC(kernel = 'linear', C = 1).fit(X_train, y_train)\n",
    "svm_predictions = svm_model_linear.predict(X_test)\n",
    "svm_accuracy = svm_model_linear.score(X_test, y_test)\n",
    "svm_cm = confusion_matrix(y_test, svm_predictions)\n",
    "print(svm_accuracy)\n",
    "print('\\n')\n",
    "print(svm_cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "# One-vs-Rest classification\n",
    "OVR_model = OneVsRestClassifier(LinearSVC(random_state=0), n_jobs = -1).fit(X_train, y_train)\n",
    "OVR_predictions = OVR_model.predict(X_test)\n",
    "OVR_accuracy = OVR_model.score(X_test, y_test)\n",
    "OVR_cm = confusion_matrix(y_test, OVR_predictions)\n",
    "print(OVR_accuracy)\n",
    "print('\\n')\n",
    "print(OVR_cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# Multiclass Logistic\n",
    "logit_model = LogisticRegression(solver = 'lbfgs', n_jobs = -1).fit(X_train, y_train)\n",
    "logit_predictions = logit_model.predict(X_test)\n",
    "logit_accuracy = logit_model.score(X_test, y_test)\n",
    "logit_cm = confusion_matrix(y_test, logit_predictions)\n",
    "print('Logistic test accuracy:\\n', logit_accuracy)\n",
    "print('\\n')\n",
    "print('Logistic confusion matrix:\\n', logit_cm)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "df_clean_generator.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}