Entity_NB2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "import en_core_web_md\n",
    "import re\n",
    "import time\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from pandarallel import pandarallel\n",
    "\n",
    "nlp = en_core_web_md.load()\n",
    "\n",
    "RE_STRING = '[.!?\\\\-]|[0-9]'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Pandarallel will run on 32 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n"
     ]
    }
   ],
   "source": [
    "pandarallel.initialize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "philoso_df = pd.read_pickle('pickles/nb2_philoso_df.tar.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LT_18940102_ARTICLE15</th>\n",
       "      <td>ROMANTIC WOMEN.</td>\n",
       "      <td>[Most women are inclined to be romantic. This ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940102_ARTICLE26</th>\n",
       "      <td>The Lyttelton Times. TUESDAY, JAN. 2, 1894.</td>\n",
       "      <td>[Whkn previously it Las been our task to look ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940105_ARTICLE16</th>\n",
       "      <td>STATE EDUCATION.</td>\n",
       "      <td>[TO THE EDITOR. Sib, —No doubt a large majorit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940108_ARTICLE22</th>\n",
       "      <td>The Lyttelton Times. MONDAY. JAN. 8, 1894.</td>\n",
       "      <td>[Among the five bishops who passed away last y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940109_ARTICLE7</th>\n",
       "      <td>THE TASK OF THE BIOLOGIST.</td>\n",
       "      <td>[No. 11. When chemistry had finished _ shaping...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790120_ARTICLE11</th>\n",
       "      <td>THE BIBLE IN SCHOOLS.</td>\n",
       "      <td>[The Rev. Dr Roseby presched en this sub j set...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790120_ARTICLE17</th>\n",
       "      <td>THE EARLY SETTLERS AND THE BIBLE. TO THE EDITOR.</td>\n",
       "      <td>[Sir, —I was gratified to read in your leading...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790121_ARTICLE3</th>\n",
       "      <td>The Otago Daily Times. WITH WHICH IS INCORPORA...</td>\n",
       "      <td>[The Committee appointed by the Athenceum meet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790121_ARTICLE16</th>\n",
       "      <td>GODLESS EDUCATION. TO THE EDITOR</td>\n",
       "      <td>[Sir,—l clxervo that the reft-reuce to tbo \"jj...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790121_ARTICLE17</th>\n",
       "      <td>THE BIBLE SCHOOLS. TO THE EDITOR.</td>\n",
       "      <td>[Sia,—lt may not be out of place, in the prese...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44730 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    Title  \\\n",
       "LT_18940102_ARTICLE15                                     ROMANTIC WOMEN.   \n",
       "LT_18940102_ARTICLE26         The Lyttelton Times. TUESDAY, JAN. 2, 1894.   \n",
       "LT_18940105_ARTICLE16                                    STATE EDUCATION.   \n",
       "LT_18940108_ARTICLE22          The Lyttelton Times. MONDAY. JAN. 8, 1894.   \n",
       "LT_18940109_ARTICLE7                           THE TASK OF THE BIOLOGIST.   \n",
       "...                                                                   ...   \n",
       "ODT_18790120_ARTICLE11                              THE BIBLE IN SCHOOLS.   \n",
       "ODT_18790120_ARTICLE17   THE EARLY SETTLERS AND THE BIBLE. TO THE EDITOR.   \n",
       "ODT_18790121_ARTICLE3   The Otago Daily Times. WITH WHICH IS INCORPORA...   \n",
       "ODT_18790121_ARTICLE16                   GODLESS EDUCATION. TO THE EDITOR   \n",
       "ODT_18790121_ARTICLE17                  THE BIBLE SCHOOLS. TO THE EDITOR.   \n",
       "\n",
       "                                                                     Text  \n",
       "LT_18940102_ARTICLE15   [Most women are inclined to be romantic. This ...  \n",
       "LT_18940102_ARTICLE26   [Whkn previously it Las been our task to look ...  \n",
       "LT_18940105_ARTICLE16   [TO THE EDITOR. Sib, —No doubt a large majorit...  \n",
       "LT_18940108_ARTICLE22   [Among the five bishops who passed away last y...  \n",
       "LT_18940109_ARTICLE7    [No. 11. When chemistry had finished _ shaping...  \n",
       "...                                                                   ...  \n",
       "ODT_18790120_ARTICLE11  [The Rev. Dr Roseby presched en this sub j set...  \n",
       "ODT_18790120_ARTICLE17  [Sir, —I was gratified to read in your leading...  \n",
       "ODT_18790121_ARTICLE3   [The Committee appointed by the Athenceum meet...  \n",
       "ODT_18790121_ARTICLE16  [Sir,—l clxervo that the reft-reuce to tbo \"jj...  \n",
       "ODT_18790121_ARTICLE17  [Sia,—lt may not be out of place, in the prese...  \n",
       "\n",
       "[44730 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "philoso_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-615.3583090305328\n"
     ]
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "philoso_df['Proper Nouns'] = philoso_df['Text'].parallel_apply(lambda x: propn_extract(x))\n",
    "print(f'{t0 - time.time()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "      <th>Proper Nouns</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LT_18940102_ARTICLE15</th>\n",
       "      <td>ROMANTIC WOMEN.</td>\n",
       "      <td>[Most women are inclined to be romantic. This ...</td>\n",
       "      <td>{ia , atmos , ba , phere }</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940102_ARTICLE26</th>\n",
       "      <td>The Lyttelton Times. TUESDAY, JAN. 2, 1894.</td>\n",
       "      <td>[Whkn previously it Las been our task to look ...</td>\n",
       "      <td>{o£ , January , Las , temptation , Whkn , tbe ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940105_ARTICLE16</th>\n",
       "      <td>STATE EDUCATION.</td>\n",
       "      <td>[TO THE EDITOR. Sib, —No doubt a large majorit...</td>\n",
       "      <td>{”—l , aa , LEV , News , Bible , Onward , bo ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940108_ARTICLE22</th>\n",
       "      <td>The Lyttelton Times. MONDAY. JAN. 8, 1894.</td>\n",
       "      <td>[Among the five bishops who passed away last y...</td>\n",
       "      <td>{Trinity , tho , himaelf , Chrysostom , Ho , B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940109_ARTICLE7</th>\n",
       "      <td>THE TASK OF THE BIOLOGIST.</td>\n",
       "      <td>[No. 11. When chemistry had finished _ shaping...</td>\n",
       "      <td>{archmopteryx , Ho , ne , thq , ia , comJ , th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790120_ARTICLE11</th>\n",
       "      <td>THE BIBLE IN SCHOOLS.</td>\n",
       "      <td>[The Rev. Dr Roseby presched en this sub j set...</td>\n",
       "      <td>{thsre , somrbody, kr;o«letlge , thai , truu ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790120_ARTICLE17</th>\n",
       "      <td>THE EARLY SETTLERS AND THE BIBLE. TO THE EDITOR.</td>\n",
       "      <td>[Sir, —I was gratified to read in your leading...</td>\n",
       "      <td>{London , Thomas , Sir, West , Bible , Colony ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790121_ARTICLE3</th>\n",
       "      <td>The Otago Daily Times. WITH WHICH IS INCORPORA...</td>\n",
       "      <td>[The Committee appointed by the Athenceum meet...</td>\n",
       "      <td>{loco , tho , num , State , rea , Btule , lar ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790121_ARTICLE16</th>\n",
       "      <td>GODLESS EDUCATION. TO THE EDITOR</td>\n",
       "      <td>[Sir,—l clxervo that the reft-reuce to tbo \"jj...</td>\n",
       "      <td>{wh , tho , pur , tbo , aueh , Ko , uso , deuh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790121_ARTICLE17</th>\n",
       "      <td>THE BIBLE SCHOOLS. TO THE EDITOR.</td>\n",
       "      <td>[Sia,—lt may not be out of place, in the prese...</td>\n",
       "      <td>{Land, Holy , Jesus, Assyiia , ot , Matthew , ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44730 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    Title  \\\n",
       "LT_18940102_ARTICLE15                                     ROMANTIC WOMEN.   \n",
       "LT_18940102_ARTICLE26         The Lyttelton Times. TUESDAY, JAN. 2, 1894.   \n",
       "LT_18940105_ARTICLE16                                    STATE EDUCATION.   \n",
       "LT_18940108_ARTICLE22          The Lyttelton Times. MONDAY. JAN. 8, 1894.   \n",
       "LT_18940109_ARTICLE7                           THE TASK OF THE BIOLOGIST.   \n",
       "...                                                                   ...   \n",
       "ODT_18790120_ARTICLE11                              THE BIBLE IN SCHOOLS.   \n",
       "ODT_18790120_ARTICLE17   THE EARLY SETTLERS AND THE BIBLE. TO THE EDITOR.   \n",
       "ODT_18790121_ARTICLE3   The Otago Daily Times. WITH WHICH IS INCORPORA...   \n",
       "ODT_18790121_ARTICLE16                   GODLESS EDUCATION. TO THE EDITOR   \n",
       "ODT_18790121_ARTICLE17                  THE BIBLE SCHOOLS. TO THE EDITOR.   \n",
       "\n",
       "                                                                     Text  \\\n",
       "LT_18940102_ARTICLE15   [Most women are inclined to be romantic. This ...   \n",
       "LT_18940102_ARTICLE26   [Whkn previously it Las been our task to look ...   \n",
       "LT_18940105_ARTICLE16   [TO THE EDITOR. Sib, —No doubt a large majorit...   \n",
       "LT_18940108_ARTICLE22   [Among the five bishops who passed away last y...   \n",
       "LT_18940109_ARTICLE7    [No. 11. When chemistry had finished _ shaping...   \n",
       "...                                                                   ...   \n",
       "ODT_18790120_ARTICLE11  [The Rev. Dr Roseby presched en this sub j set...   \n",
       "ODT_18790120_ARTICLE17  [Sir, —I was gratified to read in your leading...   \n",
       "ODT_18790121_ARTICLE3   [The Committee appointed by the Athenceum meet...   \n",
       "ODT_18790121_ARTICLE16  [Sir,—l clxervo that the reft-reuce to tbo \"jj...   \n",
       "ODT_18790121_ARTICLE17  [Sia,—lt may not be out of place, in the prese...   \n",
       "\n",
       "                                                             Proper Nouns  \n",
       "LT_18940102_ARTICLE15                          {ia , atmos , ba , phere }  \n",
       "LT_18940102_ARTICLE26   {o£ , January , Las , temptation , Whkn , tbe ...  \n",
       "LT_18940105_ARTICLE16   {”—l , aa , LEV , News , Bible , Onward , bo ,...  \n",
       "LT_18940108_ARTICLE22   {Trinity , tho , himaelf , Chrysostom , Ho , B...  \n",
       "LT_18940109_ARTICLE7    {archmopteryx , Ho , ne , thq , ia , comJ , th...  \n",
       "...                                                                   ...  \n",
       "ODT_18790120_ARTICLE11  {thsre , somrbody, kr;o«letlge , thai , truu ,...  \n",
       "ODT_18790120_ARTICLE17  {London , Thomas , Sir, West , Bible , Colony ...  \n",
       "ODT_18790121_ARTICLE3   {loco , tho , num , State , rea , Btule , lar ...  \n",
       "ODT_18790121_ARTICLE16  {wh , tho , pur , tbo , aueh , Ko , uso , deuh...  \n",
       "ODT_18790121_ARTICLE17  {Land, Holy , Jesus, Assyiia , ot , Matthew , ...  \n",
       "\n",
       "[44730 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "philoso_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "philoso_df.to_pickle('pickles/nb2_philoso_propn_df.tar.gz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Starting again for entities (should make a function to do both)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "philoso_df = pd.read_pickle('pickles/nb2_philoso_df.tar.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-625.5614910125732\n"
     ]
    }
   ],
   "source": [
    "t0 = time.time()\n",
    "philoso_df['Entities'] = philoso_df['Text'].parallel_apply(lambda x: entity_extract(x))\n",
    "print(f'{t0 - time.time()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "      <th>Entities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LT_18940102_ARTICLE15</th>\n",
       "      <td>ROMANTIC WOMEN.</td>\n",
       "      <td>[Most women are inclined to be romantic. This ...</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940102_ARTICLE26</th>\n",
       "      <td>The Lyttelton Times. TUESDAY, JAN. 2, 1894.</td>\n",
       "      <td>[Whkn previously it Las been our task to look ...</td>\n",
       "      <td>{Whkn, Las, one, the year, a year, year, last ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940105_ARTICLE16</th>\n",
       "      <td>STATE EDUCATION.</td>\n",
       "      <td>[TO THE EDITOR. Sib, —No doubt a large majorit...</td>\n",
       "      <td>{Anglicsns, the Church News, Onward New Zealan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940108_ARTICLE22</th>\n",
       "      <td>The Lyttelton Times. MONDAY. JAN. 8, 1894.</td>\n",
       "      <td>[Among the five bishops who passed away last y...</td>\n",
       "      <td>{democratic, Ho, daily, the day, Constantinopl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LT_18940109_ARTICLE7</th>\n",
       "      <td>THE TASK OF THE BIOLOGIST.</td>\n",
       "      <td>[No. 11. When chemistry had finished _ shaping...</td>\n",
       "      <td>{earth, Ho, three, first, aj reptile’s, The “ ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790120_ARTICLE11</th>\n",
       "      <td>THE BIBLE IN SCHOOLS.</td>\n",
       "      <td>[The Rev. Dr Roseby presched en this sub j set...</td>\n",
       "      <td>{tho shop, Tlio State, Proverbj, ins'ancn, Dr ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790120_ARTICLE17</th>\n",
       "      <td>THE EARLY SETTLERS AND THE BIBLE. TO THE EDITOR.</td>\n",
       "      <td>[Sir, —I was gratified to read in your leading...</td>\n",
       "      <td>{Thomas Coull, Christian, the other evening, L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790121_ARTICLE3</th>\n",
       "      <td>The Otago Daily Times. WITH WHICH IS INCORPORA...</td>\n",
       "      <td>[The Committee appointed by the Athenceum meet...</td>\n",
       "      <td>{Satur days and, Btule, five days in the week,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790121_ARTICLE16</th>\n",
       "      <td>GODLESS EDUCATION. TO THE EDITOR</td>\n",
       "      <td>[Sir,—l clxervo that the reft-reuce to tbo \"jj...</td>\n",
       "      <td>{uso, bien dis pi«cesi, mada, uttonti'jn, Ko f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18790121_ARTICLE17</th>\n",
       "      <td>THE BIBLE SCHOOLS. TO THE EDITOR.</td>\n",
       "      <td>[Sia,—lt may not be out of place, in the prese...</td>\n",
       "      <td>{thera, Ity, Freethinker, the day, Ihe Bible, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44730 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    Title  \\\n",
       "LT_18940102_ARTICLE15                                     ROMANTIC WOMEN.   \n",
       "LT_18940102_ARTICLE26         The Lyttelton Times. TUESDAY, JAN. 2, 1894.   \n",
       "LT_18940105_ARTICLE16                                    STATE EDUCATION.   \n",
       "LT_18940108_ARTICLE22          The Lyttelton Times. MONDAY. JAN. 8, 1894.   \n",
       "LT_18940109_ARTICLE7                           THE TASK OF THE BIOLOGIST.   \n",
       "...                                                                   ...   \n",
       "ODT_18790120_ARTICLE11                              THE BIBLE IN SCHOOLS.   \n",
       "ODT_18790120_ARTICLE17   THE EARLY SETTLERS AND THE BIBLE. TO THE EDITOR.   \n",
       "ODT_18790121_ARTICLE3   The Otago Daily Times. WITH WHICH IS INCORPORA...   \n",
       "ODT_18790121_ARTICLE16                   GODLESS EDUCATION. TO THE EDITOR   \n",
       "ODT_18790121_ARTICLE17                  THE BIBLE SCHOOLS. TO THE EDITOR.   \n",
       "\n",
       "                                                                     Text  \\\n",
       "LT_18940102_ARTICLE15   [Most women are inclined to be romantic. This ...   \n",
       "LT_18940102_ARTICLE26   [Whkn previously it Las been our task to look ...   \n",
       "LT_18940105_ARTICLE16   [TO THE EDITOR. Sib, —No doubt a large majorit...   \n",
       "LT_18940108_ARTICLE22   [Among the five bishops who passed away last y...   \n",
       "LT_18940109_ARTICLE7    [No. 11. When chemistry had finished _ shaping...   \n",
       "...                                                                   ...   \n",
       "ODT_18790120_ARTICLE11  [The Rev. Dr Roseby presched en this sub j set...   \n",
       "ODT_18790120_ARTICLE17  [Sir, —I was gratified to read in your leading...   \n",
       "ODT_18790121_ARTICLE3   [The Committee appointed by the Athenceum meet...   \n",
       "ODT_18790121_ARTICLE16  [Sir,—l clxervo that the reft-reuce to tbo \"jj...   \n",
       "ODT_18790121_ARTICLE17  [Sia,—lt may not be out of place, in the prese...   \n",
       "\n",
       "                                                                 Entities  \n",
       "LT_18940102_ARTICLE15                                                  {}  \n",
       "LT_18940102_ARTICLE26   {Whkn, Las, one, the year, a year, year, last ...  \n",
       "LT_18940105_ARTICLE16   {Anglicsns, the Church News, Onward New Zealan...  \n",
       "LT_18940108_ARTICLE22   {democratic, Ho, daily, the day, Constantinopl...  \n",
       "LT_18940109_ARTICLE7    {earth, Ho, three, first, aj reptile’s, The “ ...  \n",
       "...                                                                   ...  \n",
       "ODT_18790120_ARTICLE11  {tho shop, Tlio State, Proverbj, ins'ancn, Dr ...  \n",
       "ODT_18790120_ARTICLE17  {Thomas Coull, Christian, the other evening, L...  \n",
       "ODT_18790121_ARTICLE3   {Satur days and, Btule, five days in the week,...  \n",
       "ODT_18790121_ARTICLE16  {uso, bien dis pi«cesi, mada, uttonti'jn, Ko f...  \n",
       "ODT_18790121_ARTICLE17  {thera, Ity, Freethinker, the day, Ihe Bible, ...  \n",
       "\n",
       "[44730 rows x 3 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "philoso_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "philoso_df.to_pickle('pickles/nb2_philoso_entities_df.tar.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dog = nlp(' '.join(philoso_df['Text'][400]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NELSON',\n",
       " 'TUESDAY',\n",
       " 'JULY ',\n",
       " 'HON',\n",
       " 'MR',\n",
       " '. ',\n",
       " 'STAFFORD',\n",
       " 'Parliament ',\n",
       " 'New ',\n",
       " 'Zealand ',\n",
       " 'Opposition ',\n",
       " 'Mr. ',\n",
       " 'Stafford ',\n",
       " 'Opposition ',\n",
       " 'Mr. ',\n",
       " 'Stafford ',\n",
       " 'Constitution ',\n",
       " 'Act ',\n",
       " '-',\n",
       " 'Maori',\n",
       " '| ',\n",
       " 'Public ',\n",
       " 'Debts ',\n",
       " 'Act',\n",
       " 'Crown ',\n",
       " 'Mr. ',\n",
       " 'Stafford ',\n",
       " 'Governor',\n",
       " 'Minis ',\n",
       " 'ter',\n",
       " 'Blue ',\n",
       " 'Books',\n",
       " 'Mr. ',\n",
       " 'Stafford',\n",
       " 'Blue ',\n",
       " 'Book ',\n",
       " 'vol',\n",
       " '. ',\n",
       " 'D. ',\n",
       " 'Mr. ',\n",
       " 'Stafford',\n",
       " 'Mr. ',\n",
       " 'Disraeli',\n",
       " 'Parliament',\n",
       " 'Mr. ',\n",
       " 'Disraeli ',\n",
       " 'Mr. ',\n",
       " 'Glad',\n",
       " 'Ministry',\n",
       " 'British ',\n",
       " 'House ',\n",
       " 'Commons',\n",
       " 'Mr. ',\n",
       " 'Gladstone',\n",
       " 'hsw ',\n",
       " 'hs ',\n",
       " 'biffl',\n",
       " 'effec ',\n",
       " 'tive ',\n",
       " 'New ',\n",
       " 'Zealand ',\n",
       " 'Constitution',\n",
       " 'Mr. ',\n",
       " 'Disraeli',\n",
       " 'tem ',\n",
       " 'porarily ',\n",
       " 'Mr. ',\n",
       " 'Stafford',\n",
       " 'becks',\n",
       " 'arid ',\n",
       " 'Grattane']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[tok.string for tok in dog if tok.pos_ == \"PROPN\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NELSON, TUESDAY, JULY 4, 1871. A HINT FOR THE HON. MR. STAFFORD. The Parliament of New Zealand will assemble in the course of a few weeks, and, doubtless some new mode of action will be required by the leader of the Opposition that is, or is to be. Last year, in the face of a coming election, Mr. Stafford modestly withdrew from his customary prominence, preferring to put to the front a somewhat rash but unsuspecting gentleman, who, during the session, performed the Opposition talking, and, in the election which followed, lost his seat. Mr. Stafford has tried so many political movements for popularity, 1 that most of them must now be somewhat stale, and can hardly again be decently employed. It requires invention to provide novelties. The extreme radicalism be avowed when the Constitution Act wag being considered, led a large section of the public to suppose that he was the man for i the people. That served its turn for the moment, and, of course, was duly abandoned. Of his anti-Maori, and pro-war beliefs and actions; his fondness for the \" arbitrament of the sword,\" we are still \" enjoying,\" and | will long, perforce, enjoy many ofthefruitß, which will set our children's teeth on edge. His cry about economy, his promised la?ge 1 reductions which bore him bravely into ; office, and resulted in no saving at all, but great loss, and in the yet unfathomed Public Debts Act, and kindred calamities, are still ■ fresh in the memory of the public. None, even of that local band of literary admirers, ' of which he is himself supposed to be the chief, has a single word to say in defence or palliation of the most extraordinary 1 proceedings in which a modern Crown minister was ever concerned—the proceed ings of Mr. Stafford with reference to the lease, furnishings, and occupancy of the Governor's temporary residence and Minis ter's house. This business, as set forth in the Blue Books, exhibited such an amount of tricky shiftiness and misrepresentation, that a perusal of the papers would greatly tend to deter even supporters from placing reliance on either the common judgment or political fairness, to go no further, of the politician who makes such a sad figure in that correspondence. To employ what we believe is a prominent item in Mr. Stafford's own creed, that \"little arrangement\" was \" worse than a crime ;—it was a blunder.\" Unhappily for him, it was one of thoße personal blunders which, more than merely political errors, attract attention, and are treasured in the memory. The subject, as giving a gleam of character, is worth attention; and no doubt will send fresh members to study the Blue Book for 1869, vol. 2, D. 26. This, however, was not the hint we pur posed offering to Mr. Stafford, as to him it would be superfluous, seeing he is far better acquainted with all the facts than any blue book can make him. But he might take a leaf out of Mr. Disraeli's book, and, after the meeting of Parliament, rise to a little vituperation, linked at rare intervals by a few statements of fact of the kind that lurks in financial fancy. Mr. Disraeli lately made a keen and \" brilliant \" attack on Mr. Glad* stone and the work of his Ministry, and sonifl effective use might be made, in appearance at least, of the style of patriotic indignation and invective displayed by the leader of the opposition in the British House of Commons, who spoke thus : — \" On the recommendation of Mr. Gladstone, w« have legalised confiscation, we have consecrated sacrilege, we have condoned high treason, we hsw destroyed ohurches, we have shaken property to it' foundation, and we have emptied gaols; and now we cannot govern a county, and must come to. a parliamentary committee. He had found, as hs supposed, the philosopher's stone in relation to Irish affairs, and no time and labor had been grudged biffl, and no offence taken at the most startling measure'! and now the Ri.irht Honorable gentleman, after all his heroic exploits and at the head of his gM»' majority, is making government ridiculous.\" Here is a very fair model; a good, healthy, hyperbolic piece of oratory, the adoption of which, with a little alteration to suit difference of latitude, might prove as effec tive as the \"cradle and hearse\" speech which was weepingly delivered over the imaginary corpus of the New Zealand Constitution. The best point in such matters is that, as in the case of Mr. Disraeli) the speaker is gratified, the party is tem porarily pleased, and nobody is a farthing the worse. It is worthy of Mr. Stafford's consideration; and perhaps we may be allowed, should he adopt this friendly hint, to suggest \"above all,--no tears.\" That fountain i« played put, and is only provocative of \" boos, and becks' arid -wreathed smiles,\" to which are added the sneering titters of those stony ■ and selfish creatures who do not believe n> mock Grattane, and who lack faith in public display! of painfully patriotic hydraulics."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def entity_extract(strings):\n",
    "    entities = [i.text for i in nlp(' '.join(strings)).ents]\n",
    "    entities = set(entities)\n",
    "    unwanted = []\n",
    "    for entity in entities:\n",
    "        if re.search(RE_STRING, entity):\n",
    "            unwanted.append(entity)\n",
    "    entities = entities.difference(unwanted)\n",
    "    return entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def propn_extract(strings):\n",
    "    propns = [tok.string for tok in nlp(' '.join(strings)) if tok.pos_ == \"PROPN\"]\n",
    "    propns = set(propns)\n",
    "    unwanted = []\n",
    "    for prop in propns:\n",
    "        if len(prop) < 3 or re.search(RE_STRING, prop):\n",
    "            unwanted.append(prop)\n",
    "    propns = propns.difference(unwanted)\n",
    "    return propns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "philoso_df.to_pickle('pickles/philoso_propn_df.tar.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "philoso_df['Newspaper'] = philoso_df.index.map(lambda x: x[0:x.find('_')])\n",
    "philoso_df['Date']=philoso_df.index.map(lambda x: x[x.find('_')+1:x.find('_')+9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "philoso_df = philoso_df[['Title', 'Text', 'Proper Nouns']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Text</th>\n",
       "      <th>Proper Nouns</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>NA_18870416_ARTICLE4</th>\n",
       "      <td>FOEEIGN AGGRESSION.</td>\n",
       "      <td>[Fob some time past the colonial defences, the...</td>\n",
       "      <td>{Ac , compet , thiat , loverty , Mansion , ihg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NA_18870813_ARTICLE27</th>\n",
       "      <td>WHANGAREI MUTUAL IMPROVEMENT ASSOCIATION.</td>\n",
       "      <td>[On Friday evening the debate \" Moral Suasion\"...</td>\n",
       "      <td>{ho , Carruth , b|e , Brooks , Sealey , Jjecti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NA_18871022_ARTICLE5</th>\n",
       "      <td>The Northern Advocate.</td>\n",
       "      <td>[OUR COAL FIELDS. y, Ont Thursday last the Hon...</td>\n",
       "      <td>{Government , Point, Whan , Island, Hector, iA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NA_18880204_ARTICLE21</th>\n",
       "      <td>HEART WISDON.</td>\n",
       "      <td>[Unnumbered years have sped since first The bu...</td>\n",
       "      <td>{vaia, ua , Kly, Springs , Life, Becauseljto ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NA_18880623_ARTICLE29</th>\n",
       "      <td>OUR AUCKLAND LETTER. I</td>\n",
       "      <td>[« (From our own Correspondent.) Auckland, Tue...</td>\n",
       "      <td>{Saturday, Majesty, Victorian , House , ths , ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18980917_ARTICLE15</th>\n",
       "      <td>BOOK NOTICES.</td>\n",
       "      <td>[\" Corleone.\" By F. Marion Crawford. London: M...</td>\n",
       "      <td>{Bliss , Library, Corona , Lizzie , Creator , ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18980924_ARTICLE5</th>\n",
       "      <td>PASSING- NOTES.</td>\n",
       "      <td>[It is difficult to believe, probably nobody, ...</td>\n",
       "      <td>{Grey, Cathedral , Governor, Brown, Dance , no...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18981008_ARTICLE15</th>\n",
       "      <td>PUBLISHED BY SPECIAL ARRANGEMENT. WILD EELLIN ...</td>\n",
       "      <td>[COPYRIGHT., CHAPTER XXXIIL—\"SEI MIR GEG-, RUE...</td>\n",
       "      <td>{calceolarias , liavo , Somerled , Lord , Arch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18981013_ARTICLE51</th>\n",
       "      <td>BISMARCK.</td>\n",
       "      <td>[THE ATTEMPT OP AN ESTIMATE, ,■'•,' (By O. E. ...</td>\n",
       "      <td>{Goethe , Cliurch, Protestant, Frederick , sio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ODT_18981015_ARTICLE4</th>\n",
       "      <td>PASSING NOTES.</td>\n",
       "      <td>[j'n the Land of the Willow-pattern Plate ther...</td>\n",
       "      <td>{Hindostan , ia , Brown, Plate , Mongol , Japa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29647 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    Title  \\\n",
       "NA_18870416_ARTICLE4                                  FOEEIGN AGGRESSION.   \n",
       "NA_18870813_ARTICLE27           WHANGAREI MUTUAL IMPROVEMENT ASSOCIATION.   \n",
       "NA_18871022_ARTICLE5                               The Northern Advocate.   \n",
       "NA_18880204_ARTICLE21                                       HEART WISDON.   \n",
       "NA_18880623_ARTICLE29                              OUR AUCKLAND LETTER. I   \n",
       "...                                                                   ...   \n",
       "ODT_18980917_ARTICLE15                                      BOOK NOTICES.   \n",
       "ODT_18980924_ARTICLE5                                     PASSING- NOTES.   \n",
       "ODT_18981008_ARTICLE15  PUBLISHED BY SPECIAL ARRANGEMENT. WILD EELLIN ...   \n",
       "ODT_18981013_ARTICLE51                                          BISMARCK.   \n",
       "ODT_18981015_ARTICLE4                                      PASSING NOTES.   \n",
       "\n",
       "                                                                     Text  \\\n",
       "NA_18870416_ARTICLE4    [Fob some time past the colonial defences, the...   \n",
       "NA_18870813_ARTICLE27   [On Friday evening the debate \" Moral Suasion\"...   \n",
       "NA_18871022_ARTICLE5    [OUR COAL FIELDS. y, Ont Thursday last the Hon...   \n",
       "NA_18880204_ARTICLE21   [Unnumbered years have sped since first The bu...   \n",
       "NA_18880623_ARTICLE29   [« (From our own Correspondent.) Auckland, Tue...   \n",
       "...                                                                   ...   \n",
       "ODT_18980917_ARTICLE15  [\" Corleone.\" By F. Marion Crawford. London: M...   \n",
       "ODT_18980924_ARTICLE5   [It is difficult to believe, probably nobody, ...   \n",
       "ODT_18981008_ARTICLE15  [COPYRIGHT., CHAPTER XXXIIL—\"SEI MIR GEG-, RUE...   \n",
       "ODT_18981013_ARTICLE51  [THE ATTEMPT OP AN ESTIMATE, ,■'•,' (By O. E. ...   \n",
       "ODT_18981015_ARTICLE4   [j'n the Land of the Willow-pattern Plate ther...   \n",
       "\n",
       "                                                             Proper Nouns  \n",
       "NA_18870416_ARTICLE4    {Ac , compet , thiat , loverty , Mansion , ihg...  \n",
       "NA_18870813_ARTICLE27   {ho , Carruth , b|e , Brooks , Sealey , Jjecti...  \n",
       "NA_18871022_ARTICLE5    {Government , Point, Whan , Island, Hector, iA...  \n",
       "NA_18880204_ARTICLE21   {vaia, ua , Kly, Springs , Life, Becauseljto ,...  \n",
       "NA_18880623_ARTICLE29   {Saturday, Majesty, Victorian , House , ths , ...  \n",
       "...                                                                   ...  \n",
       "ODT_18980917_ARTICLE15  {Bliss , Library, Corona , Lizzie , Creator , ...  \n",
       "ODT_18980924_ARTICLE5   {Grey, Cathedral , Governor, Brown, Dance , no...  \n",
       "ODT_18981008_ARTICLE15  {calceolarias , liavo , Somerled , Lord , Arch...  \n",
       "ODT_18981013_ARTICLE51  {Goethe , Cliurch, Protestant, Frederick , sio...  \n",
       "ODT_18981015_ARTICLE4   {Hindostan , ia , Brown, Plate , Mongol , Japa...  \n",
       "\n",
       "[29647 rows x 3 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "philoso_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['the Mansion House',\n",
       " 'years',\n",
       " 'Un',\n",
       " 'auch',\n",
       " 'tjie mo st',\n",
       " 'vio lence',\n",
       " 'Germany',\n",
       " 'iorm',\n",
       " 'thiat labour',\n",
       " 'unclad',\n",
       " 'England',\n",
       " 'fed',\n",
       " 'Ac',\n",
       " 'Europe',\n",
       " 'ACes',\n",
       " 'md',\n",
       " 'Prance',\n",
       " 'India',\n",
       " 'indus',\n",
       " 'Ori thia',\n",
       " 'colonists',\n",
       " 'Sweden',\n",
       " 'the xercise dt',\n",
       " 'one']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(philoso_df.iloc[0]['Entities'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "map may be the wrong thing here (map replaces values)."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}