find_eda.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import  CountVectorizer\n",
    "import re\n",
    "import numpy as np\n",
    "import scipy.stats as stats\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "# Simple preprocessor\n",
    "def removeURLs(x):\n",
    "    # Remove URLs\n",
    "    x = re.sub(r'http\\S+', '', x)\n",
    "    \n",
    "    #Lowercase\n",
    "    x = x.lower()\n",
    "    return x\n",
    "\n",
    "def find_counts(data, text_col = \"tweet_text\", min_df=1, ngrams=(1,1)):   \n",
    "    \n",
    "    # We override the token_pattern in order to keep @signs and #hashtags\n",
    "    vec = CountVectorizer(      preprocessor=removeURLs,\n",
    "                                token_pattern = '[a-zA-Z0-9@#]+',\n",
    "                                stop_words='english',\n",
    "                                lowercase=True,\n",
    "                                min_df=min_df,\n",
    "                                ngram_range=ngrams,\n",
    "                                max_features=50000)\n",
    "    \n",
    "    bow = vec.fit_transform(data)\n",
    "    vocab = vec.get_feature_names()\n",
    "    tdm = pd.DataFrame(bow.toarray(), columns=vocab)\n",
    "        \n",
    "    \n",
    "    n_tokens = sum(tdm.sum())\n",
    "    n_docs = tdm.shape[0]\n",
    "    phrases = list(tdm.columns)\n",
    "    counts = pd.DataFrame(data={'Phrase': phrases, \n",
    "                                'Characters': [len(x) for x in phrases],\n",
    "                                'Terms': [x.count(' ')+1 for x in phrases],\n",
    "                                'isStopword': [x in stop_words for x in phrases],\n",
    "                                'Stem': [stemmer.stem(x) for x in phrases],\n",
    "                                'Count': tdm.sum(),\n",
    "                                'Count Pct': tdm.sum() / n_tokens,\n",
    "                                'Docs': tdm.astype(bool).sum(),\n",
    "                                'Docs Pct': tdm.astype(bool).sum() / n_docs,\n",
    "                          })\n",
    "    \n",
    "    counts = counts.sort_values(by=['Count'], ascending=False)\n",
    "    \n",
    "    print('Number of documents: {}'.format(tdm.shape[0]))\n",
    "    print('Number of word forms (terms): {}'.format(tdm.shape[1]))\n",
    "    print('Number of words (tokens): {}'.format(sum(counts['Count'])))\n",
    "    print('Mean words per document: {:.1f}'.format(sum(counts['Count']) / tdm.shape[0]))\n",
    "    print('Mean term occurance: {:.1f}'.format(np.mean(counts['Count'])))\n",
    "    print('Number of terms occuring once: {}'.format(sum(counts['Count']==1)))\n",
    "    print('Number of terms occuring <= 5 times: {}'.format(sum(counts['Count']<=5)))\n",
    "    print('Number of terms occuring <= 10 times: {}'.format(sum(counts['Count']<=10)))\n",
    "    print('Top {} words:'.format(num_words_to_print))\n",
    "    print(counts.head(num_words_to_print))\n",
    "    print('\\nBottom {} words:'.format(num_words_to_print))\n",
    "    print(counts.tail(num_words_to_print))\n",
    "    \n",
    "    \n",
    "    return tdm, vocab, counts \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_words_to_print=25\n",
    "\n",
    "def do_it(file_base, id_col=\"id\", text_col=\"tweet_text\", target_cols=[]):\n",
    "    df = pd.read_csv('data/'+file_base+'.csv')\n",
    "    \n",
    "    # Drop rows without any text\n",
    "    df = df.dropna(subset=[text_col])\n",
    "    \n",
    "    print(\"\\nUnigrams\")\n",
    "    tdm, vocab, counts = find_counts(df[text_col])\n",
    "    tdm.to_csv('out/'+file_base+'_unigram_tdm.csv', index=False)\n",
    "    counts.to_csv('out/'+file_base+'_unigram_counts.csv', index=False)\n",
    "    \n",
    "    print(\"\\nTrigrams\")\n",
    "    tdm, vocab, counts = find_counts(df[text_col], min_df=10, ngrams=(1,3))\n",
    "    tdm.to_csv('out/'+file_base+'_ngram_tdm.csv', index=False)\n",
    "    counts.to_csv('out/'+file_base+'_ngram_counts.csv', index=False)\n",
    "    \n",
    "    # Now, lets compute the counts and percentages per target level\n",
    "    for target_col in target_cols:\n",
    "        tdm_tmp = tdm\n",
    "        col_name = 'target_'+target_col\n",
    "        tdm_tmp[col_name] = df[target_col]\n",
    "        s = tdm_tmp.groupby(by=[col_name]).sum().T\n",
    "        n_levels=s.shape[1]\n",
    "        s['Total'] = s.sum(axis=1)\n",
    "        for i in range(0, n_levels):\n",
    "            new_col_name = str(s.columns[i]) + ' Pct'\n",
    "            s[new_col_name] = s.iloc[:,i] / s['Total']\n",
    "        #for i in range(0, n_levels):\n",
    "        #    new_col_name = str(s.columns[i]) + ' Pct'\n",
    "        #    delta = 0.0001 # To avoid log2 errors\n",
    "        #    s[str(s.columns[i]) + ' Odds'] = np.log2((delta +s[new_col_name])/ (delta + (1-s[new_col_name])))\n",
    "        s.to_csv('out/'+file_base+'_counts_'+target_col+'.csv', index=True)\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 4999\n",
      "Number of word forms (terms): 14621\n",
      "Number of words (tokens): 221753\n",
      "Mean words per document: 44.4\n",
      "Mean term occurance: 15.2\n",
      "Number of terms occuring once: 6524\n",
      "Number of terms occuring <= 5 times: 10764\n",
      "Number of terms occuring <= 10 times: 12067\n",
      "Top 25 words:\n",
      "              Phrase  Characters  Terms  isStopword     Stem  Count  \\\n",
      "t                  t           1      1        True        t   3207   \n",
      "s                  s           1      1        True        s   3199   \n",
      "like            like           4      1       False     like   3057   \n",
      "good            good           4      1       False     good   2441   \n",
      "taste          taste           5      1       False     tast   2326   \n",
      "flavor        flavor           6      1       False   flavor   2085   \n",
      "coffee        coffee           6      1       False    coffe   1913   \n",
      "just            just           4      1        True     just   1854   \n",
      "tea              tea           3      1       False      tea   1491   \n",
      "great          great           5      1       False    great   1463   \n",
      "really        really           6      1       False   realli   1301   \n",
      "product      product           7      1       False  product   1276   \n",
      "love            love           4      1       False     love   1097   \n",
      "little        little           6      1       False    littl   1011   \n",
      "use              use           3      1       False      use    995   \n",
      "don              don           3      1        True      don    978   \n",
      "chocolate  chocolate           9      1       False   chocol    953   \n",
      "sugar          sugar           5      1       False    sugar    913   \n",
      "make            make           4      1       False     make    895   \n",
      "sweet          sweet           5      1       False    sweet    847   \n",
      "water          water           5      1       False    water    840   \n",
      "m                  m           1      1        True        m    817   \n",
      "cup              cup           3      1       False      cup    810   \n",
      "ve                ve           2      1        True       ve    777   \n",
      "better        better           6      1       False   better    768   \n",
      "\n",
      "           Count Pct  Docs  Docs Pct  \n",
      "t           0.014462  1882  0.376475  \n",
      "s           0.014426  1723  0.344669  \n",
      "like        0.013786  1918  0.383677  \n",
      "good        0.011008  1776  0.355271  \n",
      "taste       0.010489  1627  0.325465  \n",
      "flavor      0.009402  1429  0.285857  \n",
      "coffee      0.008627   732  0.146429  \n",
      "just        0.008361  1383  0.276655  \n",
      "tea         0.006724   506  0.101220  \n",
      "great       0.006597  1198  0.239648  \n",
      "really      0.005867   985  0.197039  \n",
      "product     0.005754   906  0.181236  \n",
      "love        0.004947   868  0.173635  \n",
      "little      0.004559   791  0.158232  \n",
      "use         0.004487   761  0.152230  \n",
      "don         0.004410   790  0.158032  \n",
      "chocolate   0.004298   463  0.092619  \n",
      "sugar       0.004117   577  0.115423  \n",
      "make        0.004036   743  0.148630  \n",
      "sweet       0.003820   626  0.125225  \n",
      "water       0.003788   483  0.096619  \n",
      "m           0.003684   609  0.121824  \n",
      "cup         0.003653   476  0.095219  \n",
      "ve          0.003504   622  0.124425  \n",
      "better      0.003463   657  0.131426  \n",
      "\n",
      "Bottom 25 words:\n",
      "                        Phrase  Characters  Terms  isStopword            Stem  \\\n",
      "overlooked          overlooked          10      1       False        overlook   \n",
      "overload              overload           8      1       False        overload   \n",
      "overlapping        overlapping          11      1       False         overlap   \n",
      "overkill              overkill           8      1       False         overkil   \n",
      "overjoyed            overjoyed           9      1       False         overjoy   \n",
      "overides              overides           8      1       False          overid   \n",
      "overheated          overheated          10      1       False           overh   \n",
      "overflowing        overflowing          11      1       False        overflow   \n",
      "overflowed          overflowed          10      1       False        overflow   \n",
      "overestimate      overestimate          12      1       False       overestim   \n",
      "overeating          overeating          10      1       False            over   \n",
      "overdrive            overdrive           9      1       False          overdr   \n",
      "overdosing          overdosing          10      1       False         overdos   \n",
      "overdo                  overdo           6      1       False          overdo   \n",
      "overcharged        overcharged          11      1       False       overcharg   \n",
      "overcame              overcame           8      1       False         overcam   \n",
      "cyanocobalomin  cyanocobalomin          14      1       False  cyanocobalomin   \n",
      "overbearingly    overbearingly          13      1       False   overbearingli   \n",
      "cycling                cycling           7      1       False            cycl   \n",
      "overally              overally           8      1       False            over   \n",
      "cyclists              cyclists           8      1       False         cyclist   \n",
      "overabundance    overabundance          13      1       False       overabund   \n",
      "cynicism              cynicism           8      1       False           cynic   \n",
      "d110                      d110           4      1       False            d110   \n",
      "kroger                  kroger           6      1       False          kroger   \n",
      "\n",
      "                Count  Count Pct  Docs  Docs Pct  \n",
      "overlooked          1   0.000005     1    0.0002  \n",
      "overload            1   0.000005     1    0.0002  \n",
      "overlapping         1   0.000005     1    0.0002  \n",
      "overkill            1   0.000005     1    0.0002  \n",
      "overjoyed           1   0.000005     1    0.0002  \n",
      "overides            1   0.000005     1    0.0002  \n",
      "overheated          1   0.000005     1    0.0002  \n",
      "overflowing         1   0.000005     1    0.0002  \n",
      "overflowed          1   0.000005     1    0.0002  \n",
      "overestimate        1   0.000005     1    0.0002  \n",
      "overeating          1   0.000005     1    0.0002  \n",
      "overdrive           1   0.000005     1    0.0002  \n",
      "overdosing          1   0.000005     1    0.0002  \n",
      "overdo              1   0.000005     1    0.0002  \n",
      "overcharged         1   0.000005     1    0.0002  \n",
      "overcame            1   0.000005     1    0.0002  \n",
      "cyanocobalomin      1   0.000005     1    0.0002  \n",
      "overbearingly       1   0.000005     1    0.0002  \n",
      "cycling             1   0.000005     1    0.0002  \n",
      "overally            1   0.000005     1    0.0002  \n",
      "cyclists            1   0.000005     1    0.0002  \n",
      "overabundance       1   0.000005     1    0.0002  \n",
      "cynicism            1   0.000005     1    0.0002  \n",
      "d110                1   0.000005     1    0.0002  \n",
      "kroger              1   0.000005     1    0.0002  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 4999\n",
      "Number of word forms (terms): 3656\n",
      "Number of words (tokens): 220067\n",
      "Mean words per document: 44.0\n",
      "Mean term occurance: 60.2\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 202\n",
      "Top 25 words:\n",
      "              Phrase  Characters  Terms  isStopword     Stem  Count  \\\n",
      "t                  t           1      1        True        t   3207   \n",
      "s                  s           1      1        True        s   3199   \n",
      "like            like           4      1       False     like   3057   \n",
      "good            good           4      1       False     good   2441   \n",
      "taste          taste           5      1       False     tast   2326   \n",
      "flavor        flavor           6      1       False   flavor   2085   \n",
      "coffee        coffee           6      1       False    coffe   1913   \n",
      "just            just           4      1        True     just   1854   \n",
      "tea              tea           3      1       False      tea   1491   \n",
      "great          great           5      1       False    great   1463   \n",
      "really        really           6      1       False   realli   1301   \n",
      "product      product           7      1       False  product   1276   \n",
      "love            love           4      1       False     love   1097   \n",
      "little        little           6      1       False    littl   1011   \n",
      "use              use           3      1       False      use    995   \n",
      "don              don           3      1        True      don    978   \n",
      "don t          don t           5      2       False    don t    969   \n",
      "chocolate  chocolate           9      1       False   chocol    953   \n",
      "sugar          sugar           5      1       False    sugar    913   \n",
      "make            make           4      1       False     make    895   \n",
      "sweet          sweet           5      1       False    sweet    847   \n",
      "water          water           5      1       False    water    840   \n",
      "m                  m           1      1        True        m    817   \n",
      "cup              cup           3      1       False      cup    810   \n",
      "ve                ve           2      1        True       ve    777   \n",
      "\n",
      "           Count Pct  Docs  Docs Pct  \n",
      "t           0.014573  1882  0.376475  \n",
      "s           0.014536  1723  0.344669  \n",
      "like        0.013891  1918  0.383677  \n",
      "good        0.011092  1776  0.355271  \n",
      "taste       0.010570  1627  0.325465  \n",
      "flavor      0.009474  1429  0.285857  \n",
      "coffee      0.008693   732  0.146429  \n",
      "just        0.008425  1383  0.276655  \n",
      "tea         0.006775   506  0.101220  \n",
      "great       0.006648  1198  0.239648  \n",
      "really      0.005912   985  0.197039  \n",
      "product     0.005798   906  0.181236  \n",
      "love        0.004985   868  0.173635  \n",
      "little      0.004594   791  0.158232  \n",
      "use         0.004521   761  0.152230  \n",
      "don         0.004444   790  0.158032  \n",
      "don t       0.004403   784  0.156831  \n",
      "chocolate   0.004330   463  0.092619  \n",
      "sugar       0.004149   577  0.115423  \n",
      "make        0.004067   743  0.148630  \n",
      "sweet       0.003849   626  0.125225  \n",
      "water       0.003817   483  0.096619  \n",
      "m           0.003713   609  0.121824  \n",
      "cup         0.003681   476  0.095219  \n",
      "ve          0.003531   622  0.124425  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                  Phrase  Characters  Terms  isStopword  \\\n",
      "like fruit                    like fruit          10      2       False   \n",
      "t purchase                    t purchase          10      2       False   \n",
      "far better                    far better          10      2       False   \n",
      "satisfies                      satisfies           9      1       False   \n",
      "like said                      like said           9      2       False   \n",
      "explain                          explain           7      1       False   \n",
      "satisfy sweet tooth  satisfy sweet tooth          19      3       False   \n",
      "like think                    like think          10      2       False   \n",
      "excuse                            excuse           6      1       False   \n",
      "ve eating                      ve eating           9      2       False   \n",
      "think better                think better          12      2       False   \n",
      "sauce just                    sauce just          10      2       False   \n",
      "lipton tea                    lipton tea          10      2       False   \n",
      "enjoy taste                  enjoy taste          11      2       False   \n",
      "em                                    em           2      1       False   \n",
      "elbow                              elbow           5      1       False   \n",
      "saving                            saving           6      1       False   \n",
      "earlier                          earlier           7      1       False   \n",
      "little hard                  little hard          11      2       False   \n",
      "little pricey              little pricey          13      2       False   \n",
      "drink black                  drink black          11      2       False   \n",
      "drier                              drier           5      1       False   \n",
      "drawer                            drawer           6      1       False   \n",
      "little snack                little snack          12      2       False   \n",
      "product nice                product nice          12      2       False   \n",
      "\n",
      "                                    Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "like fruit                    like fruit     10   0.000045    10     0.002  \n",
      "t purchase                     t purchas     10   0.000045    10     0.002  \n",
      "far better                      far bett     10   0.000045    10     0.002  \n",
      "satisfies                        satisfi     10   0.000045    10     0.002  \n",
      "like said                      like said     10   0.000045    10     0.002  \n",
      "explain                          explain     10   0.000045    10     0.002  \n",
      "satisfy sweet tooth  satisfy sweet tooth     10   0.000045    10     0.002  \n",
      "like think                    like think     10   0.000045    10     0.002  \n",
      "excuse                             excus     10   0.000045    10     0.002  \n",
      "ve eating                         ve eat     10   0.000045    10     0.002  \n",
      "think better                  think bett     10   0.000045    10     0.002  \n",
      "sauce just                    sauce just     10   0.000045    10     0.002  \n",
      "lipton tea                    lipton tea     10   0.000045    10     0.002  \n",
      "enjoy taste                   enjoy tast     10   0.000045    10     0.002  \n",
      "em                                    em     10   0.000045    10     0.002  \n",
      "elbow                              elbow     10   0.000045    10     0.002  \n",
      "saving                              save     10   0.000045    10     0.002  \n",
      "earlier                          earlier     10   0.000045    10     0.002  \n",
      "little hard                  little hard     10   0.000045    10     0.002  \n",
      "little pricey              little pricey     10   0.000045    10     0.002  \n",
      "drink black                  drink black     10   0.000045    10     0.002  \n",
      "drier                              drier     10   0.000045    10     0.002  \n",
      "drawer                            drawer     10   0.000045    10     0.002  \n",
      "little snack                little snack     10   0.000045    10     0.002  \n",
      "product nice                 product nic     10   0.000045    10     0.002  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"amazon_food_reviews_10\", id_col=\"reviewID\", text_col=\"reviewText\", target_cols=['overall'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 6802\n",
      "Number of word forms (terms): 17542\n",
      "Number of words (tokens): 510044\n",
      "Mean words per document: 75.0\n",
      "Mean term occurance: 29.1\n",
      "Number of terms occuring once: 6445\n",
      "Number of terms occuring <= 5 times: 11889\n",
      "Number of terms occuring <= 10 times: 13610\n",
      "Top 25 words:\n",
      "              Phrase  Characters  Terms  isStopword      Stem  Count  \\\n",
      "business    business           8      1       False      busi  12645   \n",
      "loan            loan           4      1       False      loan  10648   \n",
      "children    children           8      1       False  children   7683   \n",
      "years          years           5      1       False      year   6151   \n",
      "school        school           6      1       False    school   4647   \n",
      "old              old           3      1       False       old   4129   \n",
      "buy              buy           3      1       False       buy   4062   \n",
      "able            able           4      1       False       abl   3811   \n",
      "group          group           5      1       False     group   3726   \n",
      "family        family           6      1       False    famili   3560   \n",
      "sells          sells           5      1       False      sell   3390   \n",
      "small          small           5      1       False     small   2891   \n",
      "income        income           6      1       False     incom   2723   \n",
      "husband      husband           7      1       False   husband   2717   \n",
      "married      married           7      1       False     marri   2694   \n",
      "products    products           8      1       False   product   2537   \n",
      "selling      selling           7      1       False      sell   2447   \n",
      "purchase    purchase           8      1       False   purchas   2311   \n",
      "home            home           4      1       False      home   2307   \n",
      "community  community           9      1       False    commun   2288   \n",
      "help            help           4      1       False      help   2219   \n",
      "started      started           7      1       False     start   2162   \n",
      "year            year           4      1       False      year   2049   \n",
      "increase    increase           8      1       False   increas   2005   \n",
      "store          store           5      1       False     store   1983   \n",
      "\n",
      "           Count Pct  Docs  Docs Pct  \n",
      "business    0.024792  5317  0.781682  \n",
      "loan        0.020877  6012  0.883858  \n",
      "children    0.015063  4813  0.707586  \n",
      "years       0.012060  4253  0.625257  \n",
      "school      0.009111  2987  0.439136  \n",
      "old         0.008095  3683  0.541458  \n",
      "buy         0.007964  3061  0.450015  \n",
      "able        0.007472  2713  0.398853  \n",
      "group       0.007305  1800  0.264628  \n",
      "family      0.006980  2606  0.383123  \n",
      "sells       0.006646  2645  0.388856  \n",
      "small       0.005668  2216  0.325787  \n",
      "income      0.005339  2084  0.306380  \n",
      "husband     0.005327  1983  0.291532  \n",
      "married     0.005282  2555  0.375625  \n",
      "products    0.004974  1641  0.241253  \n",
      "selling     0.004798  2021  0.297118  \n",
      "purchase    0.004531  1817  0.267127  \n",
      "home        0.004523  1660  0.244046  \n",
      "community   0.004486  1657  0.243605  \n",
      "help        0.004351  1842  0.270803  \n",
      "started     0.004239  1857  0.273008  \n",
      "year        0.004017  1728  0.254043  \n",
      "increase    0.003931  1741  0.255954  \n",
      "store       0.003888  1195  0.175684  \n",
      "\n",
      "Bottom 25 words:\n",
      "              Phrase  Characters  Terms  isStopword       Stem  Count  \\\n",
      "kirandys    kirandys           8      1       False    kirandi      1   \n",
      "kiriangi    kiriangi           8      1       False   kiriangi      1   \n",
      "kirigiris  kirigiris           9      1       False   kirigiri      1   \n",
      "kirima        kirima           6      1       False     kirima      1   \n",
      "kiriro        kiriro           6      1       False     kiriro      1   \n",
      "kirisia      kirisia           7      1       False    kirisia      1   \n",
      "kippers      kippers           7      1       False     kipper      1   \n",
      "kipngeno    kipngeno           8      1       False   kipngeno      1   \n",
      "kiplangat  kiplangat           9      1       False  kiplangat      1   \n",
      "kipchumba  kipchumba           9      1       False  kipchumba      1   \n",
      "kinyanjui  kinyanjui           9      1       False  kinyanjui      1   \n",
      "kinyozi      kinyozi           7      1       False    kinyozi      1   \n",
      "kio              kio           3      1       False        kio      1   \n",
      "kiof            kiof           4      1       False       kiof      1   \n",
      "kipanga      kipanga           7      1       False    kipanga      1   \n",
      "kiparki      kiparki           7      1       False    kiparki      1   \n",
      "kipero        kipero           6      1       False     kipero      1   \n",
      "kipkurui    kipkurui           8      1       False   kipkurui      1   \n",
      "kipkemoi    kipkemoi           8      1       False   kipkemoi      1   \n",
      "kipkirui    kipkirui           8      1       False   kipkirui      1   \n",
      "kipkoech    kipkoech           8      1       False   kipkoech      1   \n",
      "kipkongei  kipkongei           9      1       False  kipkongei      1   \n",
      "kipkorir    kipkorir           8      1       False   kipkorir      1   \n",
      "kipkurgat  kipkurgat           9      1       False  kipkurgat      1   \n",
      "konchella  konchella           9      1       False  konchella      1   \n",
      "\n",
      "           Count Pct  Docs  Docs Pct  \n",
      "kirandys    0.000002     1  0.000147  \n",
      "kiriangi    0.000002     1  0.000147  \n",
      "kirigiris   0.000002     1  0.000147  \n",
      "kirima      0.000002     1  0.000147  \n",
      "kiriro      0.000002     1  0.000147  \n",
      "kirisia     0.000002     1  0.000147  \n",
      "kippers     0.000002     1  0.000147  \n",
      "kipngeno    0.000002     1  0.000147  \n",
      "kiplangat   0.000002     1  0.000147  \n",
      "kipchumba   0.000002     1  0.000147  \n",
      "kinyanjui   0.000002     1  0.000147  \n",
      "kinyozi     0.000002     1  0.000147  \n",
      "kio         0.000002     1  0.000147  \n",
      "kiof        0.000002     1  0.000147  \n",
      "kipanga     0.000002     1  0.000147  \n",
      "kiparki     0.000002     1  0.000147  \n",
      "kipero      0.000002     1  0.000147  \n",
      "kipkurui    0.000002     1  0.000147  \n",
      "kipkemoi    0.000002     1  0.000147  \n",
      "kipkirui    0.000002     1  0.000147  \n",
      "kipkoech    0.000002     1  0.000147  \n",
      "kipkongei   0.000002     1  0.000147  \n",
      "kipkorir    0.000002     1  0.000147  \n",
      "kipkurgat   0.000002     1  0.000147  \n",
      "konchella   0.000002     1  0.000147  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 6802\n",
      "Number of word forms (terms): 11478\n",
      "Number of words (tokens): 702773\n",
      "Mean words per document: 103.3\n",
      "Mean term occurance: 61.2\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 991\n",
      "Top 25 words:\n",
      "              Phrase  Characters  Terms  isStopword       Stem  Count  \\\n",
      "business    business           8      1       False       busi  12645   \n",
      "loan            loan           4      1       False       loan  10648   \n",
      "children    children           8      1       False   children   7683   \n",
      "years          years           5      1       False       year   6151   \n",
      "school        school           6      1       False     school   4647   \n",
      "old              old           3      1       False        old   4129   \n",
      "buy              buy           3      1       False        buy   4062   \n",
      "able            able           4      1       False        abl   3811   \n",
      "group          group           5      1       False      group   3726   \n",
      "family        family           6      1       False     famili   3560   \n",
      "sells          sells           5      1       False       sell   3390   \n",
      "small          small           5      1       False      small   2891   \n",
      "years old  years old           9      2       False  years old   2737   \n",
      "income        income           6      1       False      incom   2723   \n",
      "husband      husband           7      1       False    husband   2717   \n",
      "married      married           7      1       False      marri   2694   \n",
      "products    products           8      1       False    product   2537   \n",
      "selling      selling           7      1       False       sell   2447   \n",
      "purchase    purchase           8      1       False    purchas   2311   \n",
      "home            home           4      1       False       home   2307   \n",
      "community  community           9      1       False     commun   2288   \n",
      "help            help           4      1       False       help   2219   \n",
      "started      started           7      1       False      start   2162   \n",
      "year            year           4      1       False       year   2049   \n",
      "increase    increase           8      1       False    increas   2005   \n",
      "\n",
      "           Count Pct  Docs  Docs Pct  \n",
      "business    0.017993  5317  0.781682  \n",
      "loan        0.015151  6012  0.883858  \n",
      "children    0.010932  4813  0.707586  \n",
      "years       0.008752  4253  0.625257  \n",
      "school      0.006612  2987  0.439136  \n",
      "old         0.005875  3683  0.541458  \n",
      "buy         0.005780  3061  0.450015  \n",
      "able        0.005423  2713  0.398853  \n",
      "group       0.005302  1800  0.264628  \n",
      "family      0.005066  2606  0.383123  \n",
      "sells       0.004824  2645  0.388856  \n",
      "small       0.004114  2216  0.325787  \n",
      "years old   0.003895  2607  0.383270  \n",
      "income      0.003875  2084  0.306380  \n",
      "husband     0.003866  1983  0.291532  \n",
      "married     0.003833  2555  0.375625  \n",
      "products    0.003610  1641  0.241253  \n",
      "selling     0.003482  2021  0.297118  \n",
      "purchase    0.003288  1817  0.267127  \n",
      "home        0.003283  1660  0.244046  \n",
      "community   0.003256  1657  0.243605  \n",
      "help        0.003157  1842  0.270803  \n",
      "started     0.003076  1857  0.273008  \n",
      "year        0.002916  1728  0.254043  \n",
      "increase    0.002853  1741  0.255954  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                        Phrase  Characters  Terms  isStopword  \\\n",
      "stores markets                  stores markets          14      2       False   \n",
      "farming lacks                    farming lacks          13      2       False   \n",
      "farming farm                      farming farm          12      2       False   \n",
      "old mother boys                old mother boys          15      3       False   \n",
      "old married husband        old married husband          19      3       False   \n",
      "better income family      better income family          20      3       False   \n",
      "stylish                                stylish           7      1       False   \n",
      "finish construction        finish construction          19      2       False   \n",
      "old attend school            old attend school          17      3       False   \n",
      "old businessman                old businessman          15      2       False   \n",
      "began current                    began current          13      2       False   \n",
      "began current business  began current business          22      3       False   \n",
      "suffers                                suffers           7      1       False   \n",
      "old children boys            old children boys          17      3       False   \n",
      "financial status              financial status          16      2       False   \n",
      "old daughter husband      old daughter husband          20      3       False   \n",
      "successfully paid            successfully paid          17      2       False   \n",
      "believes new                      believes new          12      2       False   \n",
      "old grown                            old grown           9      2       False   \n",
      "best selling                      best selling          12      2       False   \n",
      "old hard                              old hard           8      2       False   \n",
      "old lady married              old lady married          16      3       False   \n",
      "fifties                                fifties           7      1       False   \n",
      "old lives children          old lives children          18      3       False   \n",
      "ksh 70                                  ksh 70           6      2       False   \n",
      "\n",
      "                                        Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "stores markets                 stores market     10   0.000014    10   0.00147  \n",
      "farming lacks                   farming lack     10   0.000014    10   0.00147  \n",
      "farming farm                    farming farm     10   0.000014    10   0.00147  \n",
      "old mother boys               old mother boy     10   0.000014    10   0.00147  \n",
      "old married husband      old married husband     10   0.000014    10   0.00147  \n",
      "better income family    better income famili     10   0.000014    10   0.00147  \n",
      "stylish                              stylish     10   0.000014    10   0.00147  \n",
      "finish construction         finish construct     10   0.000014    10   0.00147  \n",
      "old attend school          old attend school     10   0.000014    10   0.00147  \n",
      "old businessman              old businessman     10   0.000014    10   0.00147  \n",
      "began current                     began curr     10   0.000014    10   0.00147  \n",
      "began current business    began current busi     10   0.000014    10   0.00147  \n",
      "suffers                               suffer     10   0.000014    10   0.00147  \n",
      "old children boys           old children boy     10   0.000014    10   0.00147  \n",
      "financial status             financial statu     10   0.000014    10   0.00147  \n",
      "old daughter husband    old daughter husband     10   0.000014    10   0.00147  \n",
      "successfully paid          successfully paid     10   0.000014    10   0.00147  \n",
      "believes new                    believes new     10   0.000014    10   0.00147  \n",
      "old grown                          old grown     10   0.000014    10   0.00147  \n",
      "best selling                        best sel     10   0.000014    10   0.00147  \n",
      "old hard                            old hard     10   0.000014    10   0.00147  \n",
      "old lady married              old lady marri     10   0.000014    10   0.00147  \n",
      "fifties                                fifti     10   0.000014    10   0.00147  \n",
      "old lives children        old lives children     10   0.000014    10   0.00147  \n",
      "ksh 70                                ksh 70     10   0.000014    10   0.00147  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"kiva_cleaned\", id_col=\"loan_id\", text_col=\"en_clean\", target_cols=['status','country', 'gender', 'nonpayment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 3230\n",
      "Number of word forms (terms): 4145\n",
      "Number of words (tokens): 29708\n",
      "Mean words per document: 9.2\n",
      "Mean term occurance: 7.2\n",
      "Number of terms occuring once: 2001\n",
      "Number of terms occuring <= 5 times: 3342\n",
      "Number of terms occuring <= 10 times: 3675\n",
      "Top 25 words:\n",
      "                      Phrase  Characters  Terms  isStopword        Stem  \\\n",
      "president          president           9      1       False      presid   \n",
      "obama                  obama           5      1       False       obama   \n",
      "s                          s           1      1        True           s   \n",
      "rt                        rt           2      1       False          rt   \n",
      "#actonclimate  #actonclimate          13      1       False  #actonclim   \n",
      "change                change           6      1       False       chang   \n",
      "t                          t           1      1        True           t   \n",
      "watch                  watch           5      1       False       watch   \n",
      "climate              climate           7      1       False      climat   \n",
      "@whitehouse      @whitehouse          11      1       False  @whitehous   \n",
      "time                    time           4      1       False        time   \n",
      "today                  today           5      1       False       today   \n",
      "health                health           6      1       False      health   \n",
      "senate                senate           6      1       False       senat   \n",
      "americans          americans           9      1       False    american   \n",
      "america              america           7      1       False     america   \n",
      "@ofa                    @ofa           4      1       False        @ofa   \n",
      "economy              economy           7      1       False     economi   \n",
      "#doyourjob        #doyourjob          10      1       False  #doyourjob   \n",
      "make                    make           4      1       False        make   \n",
      "live                    live           4      1       False        live   \n",
      "read                    read           4      1       False        read   \n",
      "#sotu                  #sotu           5      1       False       #sotu   \n",
      "#getcovered      #getcovered          11      1       False     #getcov   \n",
      "add                      add           3      1       False         add   \n",
      "\n",
      "               Count  Count Pct  Docs  Docs Pct  \n",
      "president       1270   0.042749  1261  0.390402  \n",
      "obama           1165   0.039215  1163  0.360062  \n",
      "s                885   0.029790   796  0.246440  \n",
      "rt               376   0.012657   359  0.111146  \n",
      "#actonclimate    307   0.010334   307  0.095046  \n",
      "change           235   0.007910   229  0.070898  \n",
      "t                222   0.007473   212  0.065635  \n",
      "watch            210   0.007069   210  0.065015  \n",
      "climate          207   0.006968   199  0.061610  \n",
      "@whitehouse      201   0.006766   201  0.062229  \n",
      "time             186   0.006261   184  0.056966  \n",
      "today            179   0.006025   179  0.055418  \n",
      "health           174   0.005857   172  0.053251  \n",
      "senate           156   0.005251   156  0.048297  \n",
      "americans        156   0.005251   155  0.047988  \n",
      "america          155   0.005217   154  0.047678  \n",
      "@ofa             155   0.005217   155  0.047988  \n",
      "economy          151   0.005083   151  0.046749  \n",
      "#doyourjob       150   0.005049   150  0.046440  \n",
      "make             150   0.005049   148  0.045820  \n",
      "live             148   0.004982   147  0.045511  \n",
      "read             135   0.004544   135  0.041796  \n",
      "#sotu            125   0.004208   125  0.038700  \n",
      "#getcovered      122   0.004107   122  0.037771  \n",
      "add              119   0.004006   119  0.036842  \n",
      "\n",
      "Bottom 25 words:\n",
      "                Phrase  Characters  Terms  isStopword      Stem  Count  \\\n",
      "cadets          cadets           6      1       False     cadet      1   \n",
      "cake              cake           4      1       False      cake      1   \n",
      "mountains    mountains           9      1       False  mountain      1   \n",
      "mountain      mountain           8      1       False  mountain      1   \n",
      "mount            mount           5      1       False     mount      1   \n",
      "motivation  motivation          10      1       False     motiv      1   \n",
      "motivates    motivates           9      1       False     motiv      1   \n",
      "mothers        mothers           7      1       False    mother      1   \n",
      "mother          mother           6      1       False    mother      1   \n",
      "mota              mota           4      1       False      mota      1   \n",
      "campers        campers           7      1       False    camper      1   \n",
      "candy            candy           5      1       False     candi      1   \n",
      "montana        montana           7      1       False   montana      1   \n",
      "cap                cap           3      1       False       cap      1   \n",
      "capital        capital           7      1       False     capit      1   \n",
      "cappe            cappe           5      1       False      capp      1   \n",
      "captain        captain           7      1       False   captain      1   \n",
      "career          career           6      1       False    career      1   \n",
      "careers        careers           7      1       False    career      1   \n",
      "carefully    carefully           9      1       False      care      1   \n",
      "modernize    modernize           9      1       False    modern      1   \n",
      "cares            cares           5      1       False      care      1   \n",
      "model            model           5      1       False     model      1   \n",
      "mobilizing  mobilizing          10      1       False     mobil      1   \n",
      "zones            zones           5      1       False      zone      1   \n",
      "\n",
      "            Count Pct  Docs  Docs Pct  \n",
      "cadets       0.000034     1   0.00031  \n",
      "cake         0.000034     1   0.00031  \n",
      "mountains    0.000034     1   0.00031  \n",
      "mountain     0.000034     1   0.00031  \n",
      "mount        0.000034     1   0.00031  \n",
      "motivation   0.000034     1   0.00031  \n",
      "motivates    0.000034     1   0.00031  \n",
      "mothers      0.000034     1   0.00031  \n",
      "mother       0.000034     1   0.00031  \n",
      "mota         0.000034     1   0.00031  \n",
      "campers      0.000034     1   0.00031  \n",
      "candy        0.000034     1   0.00031  \n",
      "montana      0.000034     1   0.00031  \n",
      "cap          0.000034     1   0.00031  \n",
      "capital      0.000034     1   0.00031  \n",
      "cappe        0.000034     1   0.00031  \n",
      "captain      0.000034     1   0.00031  \n",
      "career       0.000034     1   0.00031  \n",
      "careers      0.000034     1   0.00031  \n",
      "carefully    0.000034     1   0.00031  \n",
      "modernize    0.000034     1   0.00031  \n",
      "cares        0.000034     1   0.00031  \n",
      "model        0.000034     1   0.00031  \n",
      "mobilizing   0.000034     1   0.00031  \n",
      "zones        0.000034     1   0.00031  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 3230\n",
      "Number of word forms (terms): 773\n",
      "Number of words (tokens): 29359\n",
      "Mean words per document: 9.1\n",
      "Mean term occurance: 38.0\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 75\n",
      "Top 25 words:\n",
      "                          Phrase  Characters  Terms  isStopword  \\\n",
      "president              president           9      1       False   \n",
      "obama                      obama           5      1       False   \n",
      "president obama  president obama          15      2       False   \n",
      "s                              s           1      1        True   \n",
      "rt                            rt           2      1       False   \n",
      "#actonclimate      #actonclimate          13      1       False   \n",
      "change                    change           6      1       False   \n",
      "t                              t           1      1        True   \n",
      "watch                      watch           5      1       False   \n",
      "climate                  climate           7      1       False   \n",
      "@whitehouse          @whitehouse          11      1       False   \n",
      "rt @whitehouse    rt @whitehouse          14      2       False   \n",
      "time                        time           4      1       False   \n",
      "climate change    climate change          14      2       False   \n",
      "today                      today           5      1       False   \n",
      "health                    health           6      1       False   \n",
      "americans              americans           9      1       False   \n",
      "senate                    senate           6      1       False   \n",
      "america                  america           7      1       False   \n",
      "@ofa                        @ofa           4      1       False   \n",
      "economy                  economy           7      1       False   \n",
      "make                        make           4      1       False   \n",
      "#doyourjob            #doyourjob          10      1       False   \n",
      "live                        live           4      1       False   \n",
      "read                        read           4      1       False   \n",
      "\n",
      "                            Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "president                 presid   1270   0.043258  1261  0.390402  \n",
      "obama                      obama   1165   0.039681  1163  0.360062  \n",
      "president obama  president obama   1136   0.038693  1136  0.351703  \n",
      "s                              s    885   0.030144   796  0.246440  \n",
      "rt                            rt    376   0.012807   359  0.111146  \n",
      "#actonclimate         #actonclim    307   0.010457   307  0.095046  \n",
      "change                     chang    235   0.008004   229  0.070898  \n",
      "t                              t    222   0.007562   212  0.065635  \n",
      "watch                      watch    210   0.007153   210  0.065015  \n",
      "climate                   climat    207   0.007051   199  0.061610  \n",
      "@whitehouse           @whitehous    201   0.006846   201  0.062229  \n",
      "rt @whitehouse     rt @whitehous    194   0.006608   194  0.060062  \n",
      "time                        time    186   0.006335   184  0.056966  \n",
      "climate change     climate chang    179   0.006097   177  0.054799  \n",
      "today                      today    179   0.006097   179  0.055418  \n",
      "health                    health    174   0.005927   172  0.053251  \n",
      "americans               american    156   0.005314   155  0.047988  \n",
      "senate                     senat    156   0.005314   156  0.048297  \n",
      "america                  america    155   0.005279   154  0.047678  \n",
      "@ofa                        @ofa    155   0.005279   155  0.047988  \n",
      "economy                  economi    151   0.005143   151  0.046749  \n",
      "make                        make    150   0.005109   148  0.045820  \n",
      "#doyourjob            #doyourjob    150   0.005109   150  0.046440  \n",
      "live                        live    148   0.005041   147  0.045511  \n",
      "read                        read    135   0.004598   135  0.041796  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                                                  Phrase  \\\n",
      "op ed                                                              op ed   \n",
      "deficit                                                          deficit   \n",
      "supreme court nomination                        supreme court nomination   \n",
      "obama #collegeopportunity                      obama #collegeopportunity   \n",
      "today president obama                              today president obama   \n",
      "sign petition                                              sign petition   \n",
      "plans                                                              plans   \n",
      "gained                                                            gained   \n",
      "future generations                                    future generations   \n",
      "skills                                                            skills   \n",
      "options #getcovered                                  options #getcovered   \n",
      "marketplace open                                        marketplace open   \n",
      "means                                                              means   \n",
      "check president obama                              check president obama   \n",
      "child                                                              child   \n",
      "president obama #collegeopportunity  president obama #collegeopportunity   \n",
      "companies                                                      companies   \n",
      "momentum                                                        momentum   \n",
      "effects                                                          effects   \n",
      "step forward                                                step forward   \n",
      "consider                                                        consider   \n",
      "ed                                                                    ed   \n",
      "@factsonclimate                                          @factsonclimate   \n",
      "new jobs                                                        new jobs   \n",
      "safer                                                              safer   \n",
      "\n",
      "                                     Characters  Terms  isStopword  \\\n",
      "op ed                                         5      2       False   \n",
      "deficit                                       7      1       False   \n",
      "supreme court nomination                     24      3       False   \n",
      "obama #collegeopportunity                    25      2       False   \n",
      "today president obama                        21      3       False   \n",
      "sign petition                                13      2       False   \n",
      "plans                                         5      1       False   \n",
      "gained                                        6      1       False   \n",
      "future generations                           18      2       False   \n",
      "skills                                        6      1       False   \n",
      "options #getcovered                          19      2       False   \n",
      "marketplace open                             16      2       False   \n",
      "means                                         5      1       False   \n",
      "check president obama                        21      3       False   \n",
      "child                                         5      1       False   \n",
      "president obama #collegeopportunity          35      3       False   \n",
      "companies                                     9      1       False   \n",
      "momentum                                      8      1       False   \n",
      "effects                                       7      1       False   \n",
      "step forward                                 12      2       False   \n",
      "consider                                      8      1       False   \n",
      "ed                                            2      1       False   \n",
      "@factsonclimate                              15      1       False   \n",
      "new jobs                                      8      2       False   \n",
      "safer                                         5      1       False   \n",
      "\n",
      "                                                                 Stem  Count  \\\n",
      "op ed                                                             op      10   \n",
      "deficit                                                       deficit     10   \n",
      "supreme court nomination                          supreme court nomin     10   \n",
      "obama #collegeopportunity                      obama #collegeopportun     10   \n",
      "today president obama                           today president obama     10   \n",
      "sign petition                                              sign petit     10   \n",
      "plans                                                            plan     10   \n",
      "gained                                                           gain     10   \n",
      "future generations                                       future gener     10   \n",
      "skills                                                          skill     10   \n",
      "options #getcovered                                   options #getcov     10   \n",
      "marketplace open                                     marketplace open     10   \n",
      "means                                                            mean     10   \n",
      "check president obama                           check president obama     10   \n",
      "child                                                           child     10   \n",
      "president obama #collegeopportunity  president obama #collegeopportun     10   \n",
      "companies                                                     compani     10   \n",
      "momentum                                                     momentum     10   \n",
      "effects                                                        effect     10   \n",
      "step forward                                             step forward     10   \n",
      "consider                                                       consid     10   \n",
      "ed                                                                 ed     10   \n",
      "@factsonclimate                                          @factsonclim     10   \n",
      "new jobs                                                      new job     10   \n",
      "safer                                                           safer     10   \n",
      "\n",
      "                                     Count Pct  Docs  Docs Pct  \n",
      "op ed                                 0.000341    10  0.003096  \n",
      "deficit                               0.000341    10  0.003096  \n",
      "supreme court nomination              0.000341    10  0.003096  \n",
      "obama #collegeopportunity             0.000341    10  0.003096  \n",
      "today president obama                 0.000341    10  0.003096  \n",
      "sign petition                         0.000341    10  0.003096  \n",
      "plans                                 0.000341    10  0.003096  \n",
      "gained                                0.000341    10  0.003096  \n",
      "future generations                    0.000341    10  0.003096  \n",
      "skills                                0.000341    10  0.003096  \n",
      "options #getcovered                   0.000341    10  0.003096  \n",
      "marketplace open                      0.000341    10  0.003096  \n",
      "means                                 0.000341    10  0.003096  \n",
      "check president obama                 0.000341    10  0.003096  \n",
      "child                                 0.000341    10  0.003096  \n",
      "president obama #collegeopportunity   0.000341    10  0.003096  \n",
      "companies                             0.000341    10  0.003096  \n",
      "momentum                              0.000341    10  0.003096  \n",
      "effects                               0.000341    10  0.003096  \n",
      "step forward                          0.000341    10  0.003096  \n",
      "consider                              0.000341    10  0.003096  \n",
      "ed                                    0.000341    10  0.003096  \n",
      "@factsonclimate                       0.000341    10  0.003096  \n",
      "new jobs                              0.000341    10  0.003096  \n",
      "safer                                 0.000341    10  0.003096  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"obama_tweets\", id_col=\"id\", text_col=\"tweet_text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 2819\n",
      "Number of word forms (terms): 7217\n",
      "Number of words (tokens): 26120\n",
      "Mean words per document: 9.3\n",
      "Mean term occurance: 3.6\n",
      "Number of terms occuring once: 4148\n",
      "Number of terms occuring <= 5 times: 6326\n",
      "Number of terms occuring <= 10 times: 6819\n",
      "Top 25 words:\n",
      "                    Phrase  Characters  Terms  isStopword         Stem  Count  \\\n",
      "rt                      rt           2      1       False           rt    492   \n",
      "s                        s           1      1        True            s    419   \n",
      "tesla                tesla           5      1       False        tesla    335   \n",
      "model                model           5      1       False        model    244   \n",
      "t                        t           1      1        True            t    188   \n",
      "amp                    amp           3      1       False          amp    185   \n",
      "@spacex            @spacex           7      1       False      @spacex    172   \n",
      "just                  just           4      1        True         just    162   \n",
      "@teslamotors  @teslamotors          12      1       False  @teslamotor    159   \n",
      "launch              launch           6      1       False       launch    149   \n",
      "good                  good           4      1       False         good    139   \n",
      "rocket              rocket           6      1       False       rocket    134   \n",
      "like                  like           4      1       False         like    118   \n",
      "falcon              falcon           6      1       False       falcon    108   \n",
      "car                    car           3      1       False          car    105   \n",
      "yes                    yes           3      1       False           ye    101   \n",
      "1                        1           1      1       False            1     92   \n",
      "dragon              dragon           6      1       False       dragon     87   \n",
      "@elonmusk        @elonmusk           9      1       False    @elonmusk     87   \n",
      "3                        3           1      1       False            3     86   \n",
      "n                        n           1      1       False            n     85   \n",
      "9                        9           1      1       False            9     83   \n",
      "landing            landing           7      1       False         land     82   \n",
      "new                    new           3      1       False          new     80   \n",
      "time                  time           4      1       False         time     80   \n",
      "\n",
      "              Count Pct  Docs  Docs Pct  \n",
      "rt             0.018836   487  0.172756  \n",
      "s              0.016041   387  0.137283  \n",
      "tesla          0.012825   328  0.116353  \n",
      "model          0.009342   233  0.082653  \n",
      "t              0.007198   182  0.064562  \n",
      "amp            0.007083   171  0.060660  \n",
      "@spacex        0.006585   171  0.060660  \n",
      "just           0.006202   161  0.057112  \n",
      "@teslamotors   0.006087   158  0.056048  \n",
      "launch         0.005704   137  0.048599  \n",
      "good           0.005322   138  0.048954  \n",
      "rocket         0.005130   134  0.047535  \n",
      "like           0.004518   112  0.039730  \n",
      "falcon         0.004135   106  0.037602  \n",
      "car            0.004020    97  0.034409  \n",
      "yes            0.003867   101  0.035828  \n",
      "1              0.003522    84  0.029798  \n",
      "dragon         0.003331    84  0.029798  \n",
      "@elonmusk      0.003331    87  0.030862  \n",
      "3              0.003292    84  0.029798  \n",
      "n              0.003254    81  0.028734  \n",
      "9              0.003178    82  0.029088  \n",
      "landing        0.003139    75  0.026605  \n",
      "new            0.003063    78  0.027669  \n",
      "time           0.003063    79  0.028024  \n",
      "\n",
      "Bottom 25 words:\n",
      "                            Phrase  Characters  Terms  isStopword  \\\n",
      "hop                            hop           3      1       False   \n",
      "hoover                      hoover           6      1       False   \n",
      "@passionplanet1    @passionplanet1          15      1       False   \n",
      "honing                      honing           6      1       False   \n",
      "hongqiao                  hongqiao           8      1       False   \n",
      "hong                          hong           4      1       False   \n",
      "horsepower              horsepower          10      1       False   \n",
      "@osiris26                @osiris26           9      1       False   \n",
      "@ol                            @ol           3      1       False   \n",
      "houston                    houston           7      1       False   \n",
      "@olajahr                  @olajahr           8      1       False   \n",
      "hubris                      hubris           6      1       False   \n",
      "hubble                      hubble           6      1       False   \n",
      "http                          http           4      1       False   \n",
      "@oldmanriver1800  @oldmanriver1800          16      1       False   \n",
      "@oliviawilde          @oliviawilde          12      1       False   \n",
      "housing                    housing           7      1       False   \n",
      "hotel                        hotel           5      1       False   \n",
      "houses                      houses           6      1       False   \n",
      "@oprah                      @oprah           6      1       False   \n",
      "@orbcomm                  @orbcomm           8      1       False   \n",
      "@originaltman        @originaltman          13      1       False   \n",
      "hotspot                    hotspot           7      1       False   \n",
      "hotels                      hotels           6      1       False   \n",
      "zuck                          zuck           4      1       False   \n",
      "\n",
      "                              Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "hop                            hop      1   0.000038     1  0.000355  \n",
      "hoover                      hoover      1   0.000038     1  0.000355  \n",
      "@passionplanet1    @passionplanet1      1   0.000038     1  0.000355  \n",
      "honing                        hone      1   0.000038     1  0.000355  \n",
      "hongqiao                  hongqiao      1   0.000038     1  0.000355  \n",
      "hong                          hong      1   0.000038     1  0.000355  \n",
      "horsepower                horsepow      1   0.000038     1  0.000355  \n",
      "@osiris26                @osiris26      1   0.000038     1  0.000355  \n",
      "@ol                            @ol      1   0.000038     1  0.000355  \n",
      "houston                    houston      1   0.000038     1  0.000355  \n",
      "@olajahr                  @olajahr      1   0.000038     1  0.000355  \n",
      "hubris                       hubri      1   0.000038     1  0.000355  \n",
      "hubble                       hubbl      1   0.000038     1  0.000355  \n",
      "http                          http      1   0.000038     1  0.000355  \n",
      "@oldmanriver1800  @oldmanriver1800      1   0.000038     1  0.000355  \n",
      "@oliviawilde           @oliviawild      1   0.000038     1  0.000355  \n",
      "housing                       hous      1   0.000038     1  0.000355  \n",
      "hotel                        hotel      1   0.000038     1  0.000355  \n",
      "houses                        hous      1   0.000038     1  0.000355  \n",
      "@oprah                      @oprah      1   0.000038     1  0.000355  \n",
      "@orbcomm                  @orbcomm      1   0.000038     1  0.000355  \n",
      "@originaltman        @originaltman      1   0.000038     1  0.000355  \n",
      "hotspot                    hotspot      1   0.000038     1  0.000355  \n",
      "hotels                       hotel      1   0.000038     1  0.000355  \n",
      "zuck                          zuck      1   0.000038     1  0.000355  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 2819\n",
      "Number of word forms (terms): 504\n",
      "Number of words (tokens): 13890\n",
      "Mean words per document: 4.9\n",
      "Mean term occurance: 27.6\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 67\n",
      "Top 25 words:\n",
      "                    Phrase  Characters  Terms  isStopword         Stem  Count  \\\n",
      "rt                      rt           2      1       False           rt    492   \n",
      "s                        s           1      1        True            s    419   \n",
      "tesla                tesla           5      1       False        tesla    335   \n",
      "model                model           5      1       False        model    244   \n",
      "t                        t           1      1        True            t    188   \n",
      "amp                    amp           3      1       False          amp    185   \n",
      "@spacex            @spacex           7      1       False      @spacex    172   \n",
      "just                  just           4      1        True         just    162   \n",
      "@teslamotors  @teslamotors          12      1       False  @teslamotor    159   \n",
      "launch              launch           6      1       False       launch    149   \n",
      "model s            model s           7      2       False       model     146   \n",
      "good                  good           4      1       False         good    139   \n",
      "rocket              rocket           6      1       False       rocket    134   \n",
      "like                  like           4      1       False         like    118   \n",
      "falcon              falcon           6      1       False       falcon    108   \n",
      "rt @spacex      rt @spacex          10      2       False   rt @spacex    105   \n",
      "car                    car           3      1       False          car    105   \n",
      "yes                    yes           3      1       False           ye    101   \n",
      "1                        1           1      1       False            1     92   \n",
      "@elonmusk        @elonmusk           9      1       False    @elonmusk     87   \n",
      "dragon              dragon           6      1       False       dragon     87   \n",
      "3                        3           1      1       False            3     86   \n",
      "n                        n           1      1       False            n     85   \n",
      "9                        9           1      1       False            9     83   \n",
      "landing            landing           7      1       False         land     82   \n",
      "\n",
      "              Count Pct  Docs  Docs Pct  \n",
      "rt             0.035421   487  0.172756  \n",
      "s              0.030166   387  0.137283  \n",
      "tesla          0.024118   328  0.116353  \n",
      "model          0.017567   233  0.082653  \n",
      "t              0.013535   182  0.064562  \n",
      "amp            0.013319   171  0.060660  \n",
      "@spacex        0.012383   171  0.060660  \n",
      "just           0.011663   161  0.057112  \n",
      "@teslamotors   0.011447   158  0.056048  \n",
      "launch         0.010727   137  0.048599  \n",
      "model s        0.010511   145  0.051437  \n",
      "good           0.010007   138  0.048954  \n",
      "rocket         0.009647   134  0.047535  \n",
      "like           0.008495   112  0.039730  \n",
      "falcon         0.007775   106  0.037602  \n",
      "rt @spacex     0.007559   105  0.037247  \n",
      "car            0.007559    97  0.034409  \n",
      "yes            0.007271   101  0.035828  \n",
      "1              0.006623    84  0.029798  \n",
      "@elonmusk      0.006263    87  0.030862  \n",
      "dragon         0.006263    84  0.029798  \n",
      "3              0.006192    84  0.029798  \n",
      "n              0.006120    81  0.028734  \n",
      "9              0.005976    82  0.029088  \n",
      "landing        0.005904    75  0.026605  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                      Phrase  Characters  Terms  isStopword  \\\n",
      "grid                                    grid           4      1       False   \n",
      "thrusters                          thrusters           9      1       False   \n",
      "leg                                      leg           3      1       False   \n",
      "engineering                      engineering          11      1       False   \n",
      "level                                  level           5      1       False   \n",
      "congrats                            congrats           8      1       False   \n",
      "getting                              getting           7      1       False   \n",
      "line                                    line           4      1       False   \n",
      "exciting                            exciting           8      1       False   \n",
      "rt @teslamotors model  rt @teslamotors model          21      3       False   \n",
      "competition                      competition          11      1       False   \n",
      "#tesla                                #tesla           6      1       False   \n",
      "major                                  major           5      1       False   \n",
      "com                                      com           3      1       False   \n",
      "market                                market           6      1       False   \n",
      "clear                                  clear           5      1       False   \n",
      "fuel                                    fuel           4      1       False   \n",
      "check                                  check           5      1       False   \n",
      "matter                                matter           6      1       False   \n",
      "meant                                  meant           5      1       False   \n",
      "case                                    case           4      1       False   \n",
      "value                                  value           5      1       False   \n",
      "cargo                                  cargo           5      1       False   \n",
      "rt @teslamotors tesla  rt @teslamotors tesla          21      3       False   \n",
      "longer                                longer           6      1       False   \n",
      "\n",
      "                                        Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "grid                                    grid     10    0.00072    10  0.003547  \n",
      "thrusters                           thruster     10    0.00072    10  0.003547  \n",
      "leg                                      leg     10    0.00072    10  0.003547  \n",
      "engineering                            engin     10    0.00072    10  0.003547  \n",
      "level                                  level     10    0.00072    10  0.003547  \n",
      "congrats                             congrat     10    0.00072    10  0.003547  \n",
      "getting                                  get     10    0.00072    10  0.003547  \n",
      "line                                    line     10    0.00072    10  0.003547  \n",
      "exciting                               excit     10    0.00072    10  0.003547  \n",
      "rt @teslamotors model  rt @teslamotors model     10    0.00072    10  0.003547  \n",
      "competition                         competit     10    0.00072    10  0.003547  \n",
      "#tesla                                #tesla     10    0.00072    10  0.003547  \n",
      "major                                  major     10    0.00072    10  0.003547  \n",
      "com                                      com     10    0.00072    10  0.003547  \n",
      "market                                market     10    0.00072    10  0.003547  \n",
      "clear                                  clear     10    0.00072    10  0.003547  \n",
      "fuel                                    fuel     10    0.00072    10  0.003547  \n",
      "check                                  check     10    0.00072    10  0.003547  \n",
      "matter                                matter     10    0.00072    10  0.003547  \n",
      "meant                                  meant     10    0.00072    10  0.003547  \n",
      "case                                    case     10    0.00072    10  0.003547  \n",
      "value                                   valu     10    0.00072    10  0.003547  \n",
      "cargo                                  cargo     10    0.00072    10  0.003547  \n",
      "rt @teslamotors tesla  rt @teslamotors tesla     10    0.00072    10  0.003547  \n",
      "longer                                longer     10    0.00072    10  0.003547  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"elonmusk_tweets\", id_col=\"id\", text_col=\"text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 30385\n",
      "Number of word forms (terms): 31791\n",
      "Number of words (tokens): 288450\n",
      "Mean words per document: 9.5\n",
      "Mean term occurance: 9.1\n",
      "Number of terms occuring once: 18656\n",
      "Number of terms occuring <= 5 times: 26539\n",
      "Number of terms occuring <= 10 times: 28438\n",
      "Top 25 words:\n",
      "                            Phrase  Characters  Terms  isStopword  \\\n",
      "@realdonaldtrump  @realdonaldtrump          16      1       False   \n",
      "s                                s           1      1        True   \n",
      "trump                        trump           5      1       False   \n",
      "great                        great           5      1       False   \n",
      "t                                t           1      1        True   \n",
      "thanks                      thanks           6      1       False   \n",
      "thank                        thank           5      1       False   \n",
      "president                president           9      1       False   \n",
      "donald                      donald           6      1       False   \n",
      "just                          just           4      1        True   \n",
      "obama                        obama           5      1       False   \n",
      "people                      people           6      1       False   \n",
      "america                    america           7      1       False   \n",
      "like                          like           4      1       False   \n",
      "new                            new           3      1       False   \n",
      "make                          make           4      1       False   \n",
      "run                            run           3      1       False   \n",
      "good                          good           4      1       False   \n",
      "don                            don           3      1        True   \n",
      "time                          time           4      1       False   \n",
      "country                    country           7      1       False   \n",
      "#trump2016              #trump2016          10      1       False   \n",
      "best                          best           4      1       False   \n",
      "m                                m           1      1        True   \n",
      "love                          love           4      1       False   \n",
      "\n",
      "                              Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "@realdonaldtrump  @realdonaldtrump   8516   0.029523  8448  0.278032  \n",
      "s                                s   5028   0.017431  4484  0.147573  \n",
      "trump                        trump   4529   0.015701  4295  0.141353  \n",
      "great                        great   3850   0.013347  3615  0.118973  \n",
      "t                                t   2968   0.010289  2768  0.091098  \n",
      "thanks                       thank   2129   0.007381  2117  0.069673  \n",
      "thank                        thank   1959   0.006791  1934  0.063650  \n",
      "president                   presid   1832   0.006351  1787  0.058812  \n",
      "donald                      donald   1698   0.005887  1669  0.054928  \n",
      "just                          just   1669   0.005786  1627  0.053546  \n",
      "obama                        obama   1459   0.005058  1410  0.046404  \n",
      "people                       peopl   1337   0.004635  1284  0.042258  \n",
      "america                    america   1232   0.004271  1188  0.039098  \n",
      "like                          like   1216   0.004216  1182  0.038901  \n",
      "new                            new   1201   0.004164  1164  0.038308  \n",
      "make                          make   1100   0.003813  1068  0.035149  \n",
      "run                            run   1058   0.003668   986  0.032450  \n",
      "good                          good   1056   0.003661  1029  0.033865  \n",
      "don                            don   1046   0.003626  1007  0.033141  \n",
      "time                          time   1034   0.003585   999  0.032878  \n",
      "country                    countri   1024   0.003550   999  0.032878  \n",
      "#trump2016              #trump2016    913   0.003165   912  0.030015  \n",
      "best                          best    859   0.002978   830  0.027316  \n",
      "m                                m    856   0.002968   814  0.026790  \n",
      "love                          love    856   0.002968   821  0.027020  \n",
      "\n",
      "Bottom 25 words:\n",
      "                            Phrase  Characters  Terms  isStopword  \\\n",
      "@thecubiclechick  @thecubiclechick          16      1       False   \n",
      "@thecrystashow      @thecrystashow          14      1       False   \n",
      "@thefarfar              @thefarfar          10      1       False   \n",
      "@thefulldish          @thefulldish          12      1       False   \n",
      "@thegarden              @thegarden          10      1       False   \n",
      "@theholybreadcat  @theholybreadcat          16      1       False   \n",
      "@theinvestshow      @theinvestshow          14      1       False   \n",
      "@theintensity        @theintensity          13      1       False   \n",
      "@theinquisitr        @theinquisitr          13      1       False   \n",
      "@theinfinitus1      @theinfinitus1          14      1       False   \n",
      "@theimusshow          @theimusshow          12      1       False   \n",
      "@theimpregnable    @theimpregnable          15      1       False   \n",
      "@thehunterwatts    @thehunterwatts          15      1       False   \n",
      "@thehomeofgolf      @thehomeofgolf          14      1       False   \n",
      "@theholst                @theholst           9      1       False   \n",
      "@thegine3000          @thegine3000          12      1       False   \n",
      "@bradleymenglin    @bradleymenglin          15      1       False   \n",
      "@thehickman33        @thehickman33          13      1       False   \n",
      "@thehawk4221          @thehawk4221          12      1       False   \n",
      "@thegypsysons        @thegypsysons          13      1       False   \n",
      "@theguychief          @theguychief          12      1       False   \n",
      "@thegreatgedi        @thegreatgedi          13      1       False   \n",
      "@thegolfadvisor    @thegolfadvisor          15      1       False   \n",
      "@thegnomeabides    @thegnomeabides          15      1       False   \n",
      "claus                        claus           5      1       False   \n",
      "\n",
      "                              Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "@thecubiclechick  @thecubiclechick      1   0.000003     1  0.000033  \n",
      "@thecrystashow      @thecrystashow      1   0.000003     1  0.000033  \n",
      "@thefarfar              @thefarfar      1   0.000003     1  0.000033  \n",
      "@thefulldish          @thefulldish      1   0.000003     1  0.000033  \n",
      "@thegarden              @thegarden      1   0.000003     1  0.000033  \n",
      "@theholybreadcat  @theholybreadcat      1   0.000003     1  0.000033  \n",
      "@theinvestshow      @theinvestshow      1   0.000003     1  0.000033  \n",
      "@theintensity           @theintens      1   0.000003     1  0.000033  \n",
      "@theinquisitr        @theinquisitr      1   0.000003     1  0.000033  \n",
      "@theinfinitus1      @theinfinitus1      1   0.000003     1  0.000033  \n",
      "@theimusshow          @theimusshow      1   0.000003     1  0.000033  \n",
      "@theimpregnable        @theimpregn      1   0.000003     1  0.000033  \n",
      "@thehunterwatts     @thehunterwatt      1   0.000003     1  0.000033  \n",
      "@thehomeofgolf      @thehomeofgolf      1   0.000003     1  0.000033  \n",
      "@theholst                @theholst      1   0.000003     1  0.000033  \n",
      "@thegine3000          @thegine3000      1   0.000003     1  0.000033  \n",
      "@bradleymenglin    @bradleymenglin      1   0.000003     1  0.000033  \n",
      "@thehickman33        @thehickman33      1   0.000003     1  0.000033  \n",
      "@thehawk4221          @thehawk4221      1   0.000003     1  0.000033  \n",
      "@thegypsysons         @thegypsyson      1   0.000003     1  0.000033  \n",
      "@theguychief          @theguychief      1   0.000003     1  0.000033  \n",
      "@thegreatgedi        @thegreatgedi      1   0.000003     1  0.000033  \n",
      "@thegolfadvisor    @thegolfadvisor      1   0.000003     1  0.000033  \n",
      "@thegnomeabides      @thegnomeabid      1   0.000003     1  0.000033  \n",
      "claus                         clau      1   0.000003     1  0.000033  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 30385\n",
      "Number of word forms (terms): 5351\n",
      "Number of words (tokens): 282704\n",
      "Mean words per document: 9.3\n",
      "Mean term occurance: 52.8\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 442\n",
      "Top 25 words:\n",
      "                            Phrase  Characters  Terms  isStopword  \\\n",
      "@realdonaldtrump  @realdonaldtrump          16      1       False   \n",
      "s                                s           1      1        True   \n",
      "trump                        trump           5      1       False   \n",
      "great                        great           5      1       False   \n",
      "t                                t           1      1        True   \n",
      "thanks                      thanks           6      1       False   \n",
      "thank                        thank           5      1       False   \n",
      "president                president           9      1       False   \n",
      "donald                      donald           6      1       False   \n",
      "just                          just           4      1        True   \n",
      "obama                        obama           5      1       False   \n",
      "people                      people           6      1       False   \n",
      "america                    america           7      1       False   \n",
      "like                          like           4      1       False   \n",
      "new                            new           3      1       False   \n",
      "donald trump          donald trump          12      2       False   \n",
      "make                          make           4      1       False   \n",
      "run                            run           3      1       False   \n",
      "good                          good           4      1       False   \n",
      "don                            don           3      1        True   \n",
      "time                          time           4      1       False   \n",
      "country                    country           7      1       False   \n",
      "don t                        don t           5      2       False   \n",
      "#trump2016              #trump2016          10      1       False   \n",
      "best                          best           4      1       False   \n",
      "\n",
      "                              Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "@realdonaldtrump  @realdonaldtrump   8516   0.030123  8448  0.278032  \n",
      "s                                s   5028   0.017785  4484  0.147573  \n",
      "trump                        trump   4529   0.016020  4295  0.141353  \n",
      "great                        great   3850   0.013618  3615  0.118973  \n",
      "t                                t   2968   0.010499  2768  0.091098  \n",
      "thanks                       thank   2129   0.007531  2117  0.069673  \n",
      "thank                        thank   1959   0.006930  1934  0.063650  \n",
      "president                   presid   1832   0.006480  1787  0.058812  \n",
      "donald                      donald   1698   0.006006  1669  0.054928  \n",
      "just                          just   1669   0.005904  1627  0.053546  \n",
      "obama                        obama   1459   0.005161  1410  0.046404  \n",
      "people                       peopl   1337   0.004729  1284  0.042258  \n",
      "america                    america   1232   0.004358  1188  0.039098  \n",
      "like                          like   1216   0.004301  1182  0.038901  \n",
      "new                            new   1201   0.004248  1164  0.038308  \n",
      "donald trump          donald trump   1124   0.003976  1115  0.036696  \n",
      "make                          make   1100   0.003891  1068  0.035149  \n",
      "run                            run   1058   0.003742   986  0.032450  \n",
      "good                          good   1056   0.003735  1029  0.033865  \n",
      "don                            don   1046   0.003700  1007  0.033141  \n",
      "time                          time   1034   0.003658   999  0.032878  \n",
      "country                    countri   1024   0.003622   999  0.032878  \n",
      "don t                        don t    980   0.003467   941  0.030969  \n",
      "#trump2016              #trump2016    913   0.003230   912  0.030015  \n",
      "best                          best    859   0.003039   830  0.027316  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                    Phrase  Characters  Terms  isStopword  \\\n",
      "continuing                      continuing          10      1       False   \n",
      "parties                            parties           7      1       False   \n",
      "party convention          party convention          16      2       False   \n",
      "confirmed                        confirmed           9      1       False   \n",
      "concept                            concept           7      1       False   \n",
      "patrick                            patrick           7      1       False   \n",
      "complete total              complete total          14      2       False   \n",
      "pc                                      pc           2      1       False   \n",
      "@foxandfriends 8          @foxandfriends 8          16      2       False   \n",
      "trump gets                      trump gets          10      2       False   \n",
      "people scotland            people scotland          15      2       False   \n",
      "perseverance                  perseverance          12      1       False   \n",
      "persistence                    persistence          11      1       False   \n",
      "@foxandfriends great  @foxandfriends great          20      2       False   \n",
      "@foxandfriends int      @foxandfriends int          18      2       False   \n",
      "column                              column           6      1       False   \n",
      "perspective                    perspective          11      1       False   \n",
      "pete rose                        pete rose           9      2       False   \n",
      "petraeus                          petraeus           8      1       False   \n",
      "coldest                            coldest           7      1       False   \n",
      "picture mind                  picture mind          12      2       False   \n",
      "pictures                          pictures           8      1       False   \n",
      "trump golf links          trump golf links          16      3       False   \n",
      "trump going                    trump going          11      2       False   \n",
      "zone                                  zone           4      1       False   \n",
      "\n",
      "                                      Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "continuing                         continu     10   0.000035    10  0.000329  \n",
      "parties                              parti     10   0.000035    10  0.000329  \n",
      "party convention             party convent     10   0.000035    10  0.000329  \n",
      "confirmed                          confirm     10   0.000035    10  0.000329  \n",
      "concept                            concept     10   0.000035    10  0.000329  \n",
      "patrick                            patrick     10   0.000035    10  0.000329  \n",
      "complete total                complete tot     10   0.000035    10  0.000329  \n",
      "pc                                      pc     10   0.000035    10  0.000329  \n",
      "@foxandfriends 8          @foxandfriends 8     10   0.000035    10  0.000329  \n",
      "trump gets                       trump get     10   0.000035    10  0.000329  \n",
      "people scotland            people scotland     10   0.000035    10  0.000329  \n",
      "perseverance                      persever     10   0.000035    10  0.000329  \n",
      "persistence                        persist     10   0.000035    10  0.000329  \n",
      "@foxandfriends great  @foxandfriends great     10   0.000035    10  0.000329  \n",
      "@foxandfriends int      @foxandfriends int     10   0.000035    10  0.000329  \n",
      "column                              column     10   0.000035    10  0.000329  \n",
      "perspective                       perspect     10   0.000035    10  0.000329  \n",
      "pete rose                         pete ros     10   0.000035    10  0.000329  \n",
      "petraeus                           petraeu     10   0.000035    10  0.000329  \n",
      "coldest                            coldest     10   0.000035    10  0.000329  \n",
      "picture mind                  picture mind     10   0.000035    10  0.000329  \n",
      "pictures                            pictur     10   0.000035    10  0.000329  \n",
      "trump golf links           trump golf link     10   0.000035    10  0.000329  \n",
      "trump going                       trump go     10   0.000035    10  0.000329  \n",
      "zone                                  zone     10   0.000035    10  0.000329  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"2017_trump_tweets\", id_col=\"id\", text_col=\"tweet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 3339\n",
      "Number of word forms (terms): 6988\n",
      "Number of words (tokens): 40603\n",
      "Mean words per document: 12.2\n",
      "Mean term occurance: 5.8\n",
      "Number of terms occuring once: 3316\n",
      "Number of terms occuring <= 5 times: 5661\n",
      "Number of terms occuring <= 10 times: 6285\n",
      "Top 25 words:\n",
      "                    Phrase  Characters  Terms  isStopword          Stem  \\\n",
      "rt                      rt           2      1       False            rt   \n",
      "amp                    amp           3      1       False           amp   \n",
      "s                        s           1      1        True             s   \n",
      "@indiana          @indiana           8      1       False      @indiana   \n",
      "indiana            indiana           7      1       False       indiana   \n",
      "edc                    edc           3      1       False           edc   \n",
      "today                today           5      1       False         today   \n",
      "@firstladyin  @firstladyin          12      1       False  @firstladyin   \n",
      "state                state           5      1       False         state   \n",
      "jobs                  jobs           4      1       False           job   \n",
      "@govpencein    @govpencein          11      1       False   @govpencein   \n",
      "new                    new           3      1       False           new   \n",
      "w                        w           1      1       False             w   \n",
      "hoosiers          hoosiers           8      1       False       hoosier   \n",
      "hoosier            hoosier           7      1       False       hoosier   \n",
      "#indiana          #indiana           8      1       False      #indiana   \n",
      "great                great           5      1       False         great   \n",
      "2                        2           1      1       False             2   \n",
      "year                  year           4      1       False          year   \n",
      "1                        1           1      1       False             1   \n",
      "day                    day           3      1       False           day   \n",
      "@indot              @indot           6      1       False        @indot   \n",
      "governor          governor           8      1       False      governor   \n",
      "4                        4           1      1       False             4   \n",
      "@indiana2016  @indiana2016          12      1       False  @indiana2016   \n",
      "\n",
      "              Count  Count Pct  Docs  Docs Pct  \n",
      "rt             1210   0.029801  1210  0.362384  \n",
      "amp             875   0.021550   774  0.231806  \n",
      "s               855   0.021058   775  0.232105  \n",
      "@indiana        519   0.012782   513  0.153639  \n",
      "indiana         519   0.012782   506  0.151542  \n",
      "edc             508   0.012511   503  0.150644  \n",
      "today           325   0.008004   324  0.097035  \n",
      "@firstladyin    320   0.007881   320  0.095837  \n",
      "state           300   0.007389   286  0.085654  \n",
      "jobs            280   0.006896   275  0.082360  \n",
      "@govpencein     276   0.006798   276  0.082659  \n",
      "new             274   0.006748   252  0.075472  \n",
      "w               262   0.006453   253  0.075771  \n",
      "hoosiers        259   0.006379   256  0.076670  \n",
      "hoosier         243   0.005985   238  0.071279  \n",
      "#indiana        234   0.005763   234  0.070081  \n",
      "great           214   0.005271   207  0.061995  \n",
      "2               163   0.004014   152  0.045523  \n",
      "year            122   0.003005   115  0.034441  \n",
      "1               116   0.002857   113  0.033842  \n",
      "day             113   0.002783   106  0.031746  \n",
      "@indot          110   0.002709   107  0.032046  \n",
      "governor        109   0.002685   106  0.031746  \n",
      "4               109   0.002685    95  0.028452  \n",
      "@indiana2016    109   0.002685   109  0.032645  \n",
      "\n",
      "Bottom 25 words:\n",
      "                  Phrase  Characters  Terms  isStopword        Stem  Count  \\\n",
      "beagle            beagle           6      1       False       beagl      1   \n",
      "bequeathed    bequeathed          10      1       False    bequeath      1   \n",
      "benefiting    benefiting          10      1       False     benefit      1   \n",
      "beneath          beneath           7      1       False     beneath      1   \n",
      "bench              bench           5      1       False       bench      1   \n",
      "makeover        makeover           8      1       False      makeov      1   \n",
      "bell                bell           4      1       False        bell      1   \n",
      "male                male           4      1       False        male      1   \n",
      "behaviors      behaviors           9      1       False    behavior      1   \n",
      "begun              begun           5      1       False       begun      1   \n",
      "beg                  beg           3      1       False         beg      1   \n",
      "bees                bees           4      1       False         bee      1   \n",
      "beefing          beefing           7      1       False        beef      1   \n",
      "bee                  bee           3      1       False         bee      1   \n",
      "manifests      manifests           9      1       False    manifest      1   \n",
      "beds                beds           4      1       False         bed      1   \n",
      "manufacture  manufacture          11      1       False  manufactur      1   \n",
      "bed                  bed           3      1       False         bed      1   \n",
      "becky              becky           5      1       False       becki      1   \n",
      "beautifully  beautifully          11      1       False      beauti      1   \n",
      "bears              bears           5      1       False        bear      1   \n",
      "marina            marina           6      1       False      marina      1   \n",
      "bearing          bearing           7      1       False        bear      1   \n",
      "bean                bean           4      1       False        bean      1   \n",
      "zionsville    zionsville          10      1       False    zionsvil      1   \n",
      "\n",
      "             Count Pct  Docs  Docs Pct  \n",
      "beagle        0.000025     1  0.000299  \n",
      "bequeathed    0.000025     1  0.000299  \n",
      "benefiting    0.000025     1  0.000299  \n",
      "beneath       0.000025     1  0.000299  \n",
      "bench         0.000025     1  0.000299  \n",
      "makeover      0.000025     1  0.000299  \n",
      "bell          0.000025     1  0.000299  \n",
      "male          0.000025     1  0.000299  \n",
      "behaviors     0.000025     1  0.000299  \n",
      "begun         0.000025     1  0.000299  \n",
      "beg           0.000025     1  0.000299  \n",
      "bees          0.000025     1  0.000299  \n",
      "beefing       0.000025     1  0.000299  \n",
      "bee           0.000025     1  0.000299  \n",
      "manifests     0.000025     1  0.000299  \n",
      "beds          0.000025     1  0.000299  \n",
      "manufacture   0.000025     1  0.000299  \n",
      "bed           0.000025     1  0.000299  \n",
      "becky         0.000025     1  0.000299  \n",
      "beautifully   0.000025     1  0.000299  \n",
      "bears         0.000025     1  0.000299  \n",
      "marina        0.000025     1  0.000299  \n",
      "bearing       0.000025     1  0.000299  \n",
      "bean          0.000025     1  0.000299  \n",
      "zionsville    0.000025     1  0.000299  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 3339\n",
      "Number of word forms (terms): 954\n",
      "Number of words (tokens): 31389\n",
      "Mean words per document: 9.4\n",
      "Mean term occurance: 32.9\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 108\n",
      "Top 25 words:\n",
      "                          Phrase  Characters  Terms  isStopword  \\\n",
      "rt                            rt           2      1       False   \n",
      "amp                          amp           3      1       False   \n",
      "s                              s           1      1        True   \n",
      "indiana                  indiana           7      1       False   \n",
      "@indiana                @indiana           8      1       False   \n",
      "edc                          edc           3      1       False   \n",
      "@indiana edc        @indiana edc          12      2       False   \n",
      "rt @indiana          rt @indiana          11      2       False   \n",
      "rt @indiana edc  rt @indiana edc          15      3       False   \n",
      "today                      today           5      1       False   \n",
      "@firstladyin        @firstladyin          12      1       False   \n",
      "state                      state           5      1       False   \n",
      "jobs                        jobs           4      1       False   \n",
      "@govpencein          @govpencein          11      1       False   \n",
      "new                          new           3      1       False   \n",
      "w                              w           1      1       False   \n",
      "hoosiers                hoosiers           8      1       False   \n",
      "hoosier                  hoosier           7      1       False   \n",
      "#indiana                #indiana           8      1       False   \n",
      "rt @firstladyin  rt @firstladyin          15      2       False   \n",
      "great                      great           5      1       False   \n",
      "2                              2           1      1       False   \n",
      "year                        year           4      1       False   \n",
      "1                              1           1      1       False   \n",
      "day                          day           3      1       False   \n",
      "\n",
      "                            Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "rt                            rt   1210   0.038549  1210  0.362384  \n",
      "amp                          amp    875   0.027876   774  0.231806  \n",
      "s                              s    855   0.027239   775  0.232105  \n",
      "indiana                  indiana    519   0.016534   506  0.151542  \n",
      "@indiana                @indiana    519   0.016534   513  0.153639  \n",
      "edc                          edc    508   0.016184   503  0.150644  \n",
      "@indiana edc        @indiana edc    508   0.016184   503  0.150644  \n",
      "rt @indiana          rt @indiana    476   0.015165   476  0.142558  \n",
      "rt @indiana edc  rt @indiana edc    473   0.015069   473  0.141659  \n",
      "today                      today    325   0.010354   324  0.097035  \n",
      "@firstladyin        @firstladyin    320   0.010195   320  0.095837  \n",
      "state                      state    300   0.009557   286  0.085654  \n",
      "jobs                         job    280   0.008920   275  0.082360  \n",
      "@govpencein          @govpencein    276   0.008793   276  0.082659  \n",
      "new                          new    274   0.008729   252  0.075472  \n",
      "w                              w    262   0.008347   253  0.075771  \n",
      "hoosiers                 hoosier    259   0.008251   256  0.076670  \n",
      "hoosier                  hoosier    243   0.007742   238  0.071279  \n",
      "#indiana                #indiana    234   0.007455   234  0.070081  \n",
      "rt @firstladyin  rt @firstladyin    225   0.007168   225  0.067385  \n",
      "great                      great    214   0.006818   207  0.061995  \n",
      "2                              2    163   0.005193   152  0.045523  \n",
      "year                        year    122   0.003887   115  0.034441  \n",
      "1                              1    116   0.003696   113  0.033842  \n",
      "day                          day    113   0.003600   106  0.031746  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                Phrase  Characters  Terms  isStopword  \\\n",
      "commerce                      commerce           8      1       False   \n",
      "edc board                    edc board           9      2       False   \n",
      "edc #indiana ranks  edc #indiana ranks          18      3       False   \n",
      "200 new                        200 new           7      2       False   \n",
      "@subaru                        @subaru           7      1       False   \n",
      "supplier                      supplier           8      1       False   \n",
      "@isdagov                      @isdagov           8      1       False   \n",
      "relief                          relief           6      1       False   \n",
      "outstanding                outstanding          11      1       False   \n",
      "signed law                  signed law          10      2       False   \n",
      "family friends          family friends          14      2       False   \n",
      "pay                                pay           3      1       False   \n",
      "4 5                                4 5           3      2       False   \n",
      "state amp                    state amp           9      2       False   \n",
      "expected                      expected           8      1       False   \n",
      "hoosiers working      hoosiers working          16      2       False   \n",
      "clinic                          clinic           6      1       False   \n",
      "hope                              hope           4      1       False   \n",
      "college                        college           7      1       False   \n",
      "execs                            execs           5      1       False   \n",
      "exciting                      exciting           8      1       False   \n",
      "hosting                        hosting           7      1       False   \n",
      "student                        student           7      1       False   \n",
      "studio                          studio           6      1       False   \n",
      "rt @firstladyin w    rt @firstladyin w          17      3       False   \n",
      "\n",
      "                                 Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "commerce                      commerc     10   0.000319    10  0.002995  \n",
      "edc board                   edc board     10   0.000319    10  0.002995  \n",
      "edc #indiana ranks  edc #indiana rank     10   0.000319    10  0.002995  \n",
      "200 new                       200 new     10   0.000319    10  0.002995  \n",
      "@subaru                       @subaru     10   0.000319    10  0.002995  \n",
      "supplier                     supplier     10   0.000319    10  0.002995  \n",
      "@isdagov                     @isdagov     10   0.000319    10  0.002995  \n",
      "relief                         relief     10   0.000319    10  0.002995  \n",
      "outstanding                  outstand     10   0.000319    10  0.002995  \n",
      "signed law                 signed law     10   0.000319    10  0.002995  \n",
      "family friends          family friend     10   0.000319    10  0.002995  \n",
      "pay                               pay     10   0.000319    10  0.002995  \n",
      "4 5                               4 5     10   0.000319    10  0.002995  \n",
      "state amp                   state amp     10   0.000319    10  0.002995  \n",
      "expected                       expect     10   0.000319    10  0.002995  \n",
      "hoosiers working        hoosiers work     10   0.000319    10  0.002995  \n",
      "clinic                         clinic     10   0.000319    10  0.002995  \n",
      "hope                             hope     10   0.000319    10  0.002995  \n",
      "college                        colleg     10   0.000319    10  0.002995  \n",
      "execs                            exec     10   0.000319    10  0.002995  \n",
      "exciting                        excit     10   0.000319    10  0.002995  \n",
      "hosting                          host     10   0.000319    10  0.002995  \n",
      "student                       student     10   0.000319    10  0.002995  \n",
      "studio                         studio     10   0.000319    10  0.002995  \n",
      "rt @firstladyin w   rt @firstladyin w     10   0.000319    10  0.002995  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"pence_tweets\", id_col=\"id\", text_col=\"tweet_text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 19779\n",
      "Number of word forms (terms): 43819\n",
      "Number of words (tokens): 1754721\n",
      "Mean words per document: 88.7\n",
      "Mean term occurance: 40.0\n",
      "Number of terms occuring once: 16029\n",
      "Number of terms occuring <= 5 times: 29333\n",
      "Number of terms occuring <= 10 times: 33429\n",
      "Top 25 words:\n",
      "          Phrase  Characters  Terms  isStopword     Stem  Count  Count Pct  \\\n",
      "said        said           4      1       False     said  53019   0.030215   \n",
      "s              s           1      1        True        s  30166   0.017191   \n",
      "mln          mln           3      1       False      mln  25716   0.014655   \n",
      "dlrs        dlrs           4      1       False      dlr  20760   0.011831   \n",
      "reuter    reuter           6      1       False   reuter  19139   0.010907   \n",
      "pct          pct           3      1       False      pct  17212   0.009809   \n",
      "1              1           1      1       False        1  16849   0.009602   \n",
      "vs            vs           2      1       False       vs  14737   0.008398   \n",
      "000          000           3      1       False      000  13427   0.007652   \n",
      "year        year           4      1       False     year  12524   0.007137   \n",
      "2              2           1      1       False        2  10415   0.005935   \n",
      "billion  billion           7      1       False  billion  10288   0.005863   \n",
      "company  company           7      1       False  compani   9651   0.005500   \n",
      "u              u           1      1       False        u   9597   0.005469   \n",
      "cts          cts           3      1       False       ct   9003   0.005131   \n",
      "3              3           1      1       False        3   8379   0.004775   \n",
      "5              5           1      1       False        5   7411   0.004223   \n",
      "4              4           1      1       False        4   7256   0.004135   \n",
      "bank        bank           4      1       False     bank   7244   0.004128   \n",
      "lt            lt           2      1       False       lt   7029   0.004006   \n",
      "new          new           3      1       False      new   6527   0.003720   \n",
      "net          net           3      1       False      net   6107   0.003480   \n",
      "corp        corp           4      1       False     corp   6030   0.003436   \n",
      "1986        1986           4      1       False     1986   5871   0.003346   \n",
      "8              8           1      1       False        8   5839   0.003328   \n",
      "\n",
      "          Docs  Docs Pct  \n",
      "said     15375  0.777340  \n",
      "s        10119  0.511603  \n",
      "mln       8341  0.421710  \n",
      "dlrs      7730  0.390819  \n",
      "reuter   19076  0.964457  \n",
      "pct       6009  0.303807  \n",
      "1         7170  0.362506  \n",
      "vs        3104  0.156934  \n",
      "000       4761  0.240710  \n",
      "year      6070  0.306891  \n",
      "2         5483  0.277213  \n",
      "billion   3253  0.164467  \n",
      "company   5502  0.278174  \n",
      "u         4177  0.211184  \n",
      "cts       3430  0.173416  \n",
      "3         4729  0.239092  \n",
      "5         4444  0.224683  \n",
      "4         4050  0.204763  \n",
      "bank      2807  0.141918  \n",
      "lt        3811  0.192679  \n",
      "new       3823  0.193286  \n",
      "net       3267  0.165175  \n",
      "corp      4447  0.224834  \n",
      "1986      3443  0.174074  \n",
      "8         3544  0.179180  \n",
      "\n",
      "Bottom 25 words:\n",
      "                            Phrase  Characters  Terms  isStopword  \\\n",
      "mcrp                          mcrp           4      1       False   \n",
      "mcrib                        mcrib           5      1       False   \n",
      "mcneill                    mcneill           7      1       False   \n",
      "mcnab                        mcnab           5      1       False   \n",
      "antioquia                antioquia           9      1       False   \n",
      "meaningfully          meaningfully          12      1       False   \n",
      "antioch                    antioch           7      1       False   \n",
      "mecfint                    mecfint           7      1       False   \n",
      "mecklenburg            mecklenburg          11      1       False   \n",
      "mechton                    mechton           7      1       False   \n",
      "mechanized              mechanized          10      1       False   \n",
      "antidotes                antidotes           9      1       False   \n",
      "mechanisims            mechanisims          11      1       False   \n",
      "antidumping            antidumping          11      1       False   \n",
      "mechanic                  mechanic           8      1       False   \n",
      "meaty                        meaty           5      1       False   \n",
      "antimicrobial        antimicrobial          13      1       False   \n",
      "meatls                      meatls           6      1       False   \n",
      "antigone                  antigone           8      1       False   \n",
      "antigua                    antigua           7      1       False   \n",
      "antihemophilic      antihemophilic          14      1       False   \n",
      "antihistamine        antihistamine          13      1       False   \n",
      "antihypertensive  antihypertensive          16      1       False   \n",
      "meanwhle                  meanwhle           8      1       False   \n",
      "zy                              zy           2      1       False   \n",
      "\n",
      "                           Stem  Count     Count Pct  Docs  Docs Pct  \n",
      "mcrp                       mcrp      1  5.698912e-07     1  0.000051  \n",
      "mcrib                     mcrib      1  5.698912e-07     1  0.000051  \n",
      "mcneill                 mcneill      1  5.698912e-07     1  0.000051  \n",
      "mcnab                     mcnab      1  5.698912e-07     1  0.000051  \n",
      "antioquia             antioquia      1  5.698912e-07     1  0.000051  \n",
      "meaningfully            meaning      1  5.698912e-07     1  0.000051  \n",
      "antioch                 antioch      1  5.698912e-07     1  0.000051  \n",
      "mecfint                 mecfint      1  5.698912e-07     1  0.000051  \n",
      "mecklenburg         mecklenburg      1  5.698912e-07     1  0.000051  \n",
      "mechton                 mechton      1  5.698912e-07     1  0.000051  \n",
      "mechanized               mechan      1  5.698912e-07     1  0.000051  \n",
      "antidotes               antidot      1  5.698912e-07     1  0.000051  \n",
      "mechanisims          mechanisim      1  5.698912e-07     1  0.000051  \n",
      "antidumping            antidump      1  5.698912e-07     1  0.000051  \n",
      "mechanic                 mechan      1  5.698912e-07     1  0.000051  \n",
      "meaty                     meati      1  5.698912e-07     1  0.000051  \n",
      "antimicrobial       antimicrobi      1  5.698912e-07     1  0.000051  \n",
      "meatls                    meatl      1  5.698912e-07     1  0.000051  \n",
      "antigone                antigon      1  5.698912e-07     1  0.000051  \n",
      "antigua                 antigua      1  5.698912e-07     1  0.000051  \n",
      "antihemophilic     antihemophil      1  5.698912e-07     1  0.000051  \n",
      "antihistamine      antihistamin      1  5.698912e-07     1  0.000051  \n",
      "antihypertensive  antihypertens      1  5.698912e-07     1  0.000051  \n",
      "meanwhle                meanwhl      1  5.698912e-07     1  0.000051  \n",
      "zy                           zy      1  5.698912e-07     1  0.000051  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 19779\n",
      "Number of word forms (terms): 32359\n",
      "Number of words (tokens): 2477615\n",
      "Mean words per document: 125.3\n",
      "Mean term occurance: 76.6\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 2162\n",
      "Top 25 words:\n",
      "            Phrase  Characters  Terms  isStopword     Stem  Count  Count Pct  \\\n",
      "said          said           4      1       False     said  53019   0.021399   \n",
      "s                s           1      1        True        s  30166   0.012175   \n",
      "mln            mln           3      1       False      mln  25716   0.010379   \n",
      "dlrs          dlrs           4      1       False      dlr  20760   0.008379   \n",
      "reuter      reuter           6      1       False   reuter  19139   0.007725   \n",
      "pct            pct           3      1       False      pct  17212   0.006947   \n",
      "1                1           1      1       False        1  16849   0.006800   \n",
      "vs              vs           2      1       False       vs  14737   0.005948   \n",
      "000            000           3      1       False      000  13427   0.005419   \n",
      "year          year           4      1       False     year  12524   0.005055   \n",
      "2                2           1      1       False        2  10415   0.004204   \n",
      "billion    billion           7      1       False  billion  10288   0.004152   \n",
      "company    company           7      1       False  compani   9651   0.003895   \n",
      "u                u           1      1       False        u   9597   0.003873   \n",
      "cts            cts           3      1       False       ct   9003   0.003634   \n",
      "u s            u s           3      2       False       u    8795   0.003550   \n",
      "3                3           1      1       False        3   8379   0.003382   \n",
      "mln dlrs  mln dlrs           8      2       False  mln dlr   7980   0.003221   \n",
      "5                5           1      1       False        5   7411   0.002991   \n",
      "4                4           1      1       False        4   7256   0.002929   \n",
      "bank          bank           4      1       False     bank   7244   0.002924   \n",
      "lt              lt           2      1       False       lt   7029   0.002837   \n",
      "new            new           3      1       False      new   6527   0.002634   \n",
      "net            net           3      1       False      net   6107   0.002465   \n",
      "corp          corp           4      1       False     corp   6030   0.002434   \n",
      "\n",
      "           Docs  Docs Pct  \n",
      "said      15375  0.777340  \n",
      "s         10119  0.511603  \n",
      "mln        8341  0.421710  \n",
      "dlrs       7730  0.390819  \n",
      "reuter    19076  0.964457  \n",
      "pct        6009  0.303807  \n",
      "1          7170  0.362506  \n",
      "vs         3104  0.156934  \n",
      "000        4761  0.240710  \n",
      "year       6070  0.306891  \n",
      "2          5483  0.277213  \n",
      "billion    3253  0.164467  \n",
      "company    5502  0.278174  \n",
      "u          4177  0.211184  \n",
      "cts        3430  0.173416  \n",
      "u s        3863  0.195308  \n",
      "3          4729  0.239092  \n",
      "mln dlrs   3854  0.194853  \n",
      "5          4444  0.224683  \n",
      "4          4050  0.204763  \n",
      "bank       2807  0.141918  \n",
      "lt         3811  0.192679  \n",
      "new        3823  0.193286  \n",
      "net        3267  0.165175  \n",
      "corp       4447  0.224834  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                                Phrase  Characters  Terms  \\\n",
      "closely held                              closely held          12      2   \n",
      "operating company                    operating company          17      2   \n",
      "52 mln dlrs                                52 mln dlrs          11      3   \n",
      "reimburse                                    reimburse           9      1   \n",
      "fall yen                                      fall yen           8      2   \n",
      "falling oil                                falling oil          11      2   \n",
      "intervention central banks  intervention central banks          26      3   \n",
      "operations u                              operations u          12      2   \n",
      "collapse oil                              collapse oil          12      2   \n",
      "dlrs 18 cts                                dlrs 18 cts          11      3   \n",
      "1 6175 85                                    1 6175 85           9      3   \n",
      "1 6175                                          1 6175           6      2   \n",
      "1 61 dlrs                                    1 61 dlrs           9      3   \n",
      "coffee sugar cocoa                  coffee sugar cocoa          18      3   \n",
      "operations gain                        operations gain          15      2   \n",
      "rejected u                                  rejected u          10      2   \n",
      "intervention necessary          intervention necessary          22      2   \n",
      "said new products                    said new products          17      3   \n",
      "operation company                    operation company          17      2   \n",
      "88 cts net                                  88 cts net          10      3   \n",
      "said necessary                          said necessary          14      2   \n",
      "said near                                    said near           9      2   \n",
      "interview said                          interview said          14      2   \n",
      "interviewed                                interviewed          11      1   \n",
      "8 pct quarter                            8 pct quarter          13      3   \n",
      "\n",
      "                            isStopword                       Stem  Count  \\\n",
      "closely held                     False               closely held     10   \n",
      "operating company                False          operating compani     10   \n",
      "52 mln dlrs                      False                 52 mln dlr     10   \n",
      "reimburse                        False                   reimburs     10   \n",
      "fall yen                         False                   fall yen     10   \n",
      "falling oil                      False                falling oil     10   \n",
      "intervention central banks       False  intervention central bank     10   \n",
      "operations u                     False               operations u     10   \n",
      "collapse oil                     False               collapse oil     10   \n",
      "dlrs 18 cts                      False                 dlrs 18 ct     10   \n",
      "1 6175 85                        False                  1 6175 85     10   \n",
      "1 6175                           False                     1 6175     10   \n",
      "1 61 dlrs                        False                   1 61 dlr     10   \n",
      "coffee sugar cocoa               False         coffee sugar cocoa     10   \n",
      "operations gain                  False            operations gain     10   \n",
      "rejected u                       False                 rejected u     10   \n",
      "intervention necessary           False     intervention necessari     10   \n",
      "said new products                False           said new product     10   \n",
      "operation company                False          operation compani     10   \n",
      "88 cts net                       False                 88 cts net     10   \n",
      "said necessary                   False             said necessari     10   \n",
      "said near                        False                  said near     10   \n",
      "interview said                   False             interview said     10   \n",
      "interviewed                      False                  interview     10   \n",
      "8 pct quarter                    False              8 pct quarter     10   \n",
      "\n",
      "                            Count Pct  Docs  Docs Pct  \n",
      "closely held                 0.000004    10  0.000506  \n",
      "operating company            0.000004    10  0.000506  \n",
      "52 mln dlrs                  0.000004    10  0.000506  \n",
      "reimburse                    0.000004    10  0.000506  \n",
      "fall yen                     0.000004    10  0.000506  \n",
      "falling oil                  0.000004    10  0.000506  \n",
      "intervention central banks   0.000004    10  0.000506  \n",
      "operations u                 0.000004    10  0.000506  \n",
      "collapse oil                 0.000004    10  0.000506  \n",
      "dlrs 18 cts                  0.000004    10  0.000506  \n",
      "1 6175 85                    0.000004    10  0.000506  \n",
      "1 6175                       0.000004    10  0.000506  \n",
      "1 61 dlrs                    0.000004    10  0.000506  \n",
      "coffee sugar cocoa           0.000004    10  0.000506  \n",
      "operations gain              0.000004    10  0.000506  \n",
      "rejected u                   0.000004    10  0.000506  \n",
      "intervention necessary       0.000004    10  0.000506  \n",
      "said new products            0.000004    10  0.000506  \n",
      "operation company            0.000004    10  0.000506  \n",
      "88 cts net                   0.000004    10  0.000506  \n",
      "said necessary               0.000004    10  0.000506  \n",
      "said near                    0.000004    10  0.000506  \n",
      "interview said               0.000004    10  0.000506  \n",
      "interviewed                  0.000004    10  0.000506  \n",
      "8 pct quarter                0.000004    10  0.000506  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"reutersCSV\", id_col=\"pid\", text_col=\"doc.text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 4694\n",
      "Number of word forms (terms): 9712\n",
      "Number of words (tokens): 79351\n",
      "Mean words per document: 16.9\n",
      "Mean term occurance: 8.2\n",
      "Number of terms occuring once: 5180\n",
      "Number of terms occuring <= 5 times: 7783\n",
      "Number of terms occuring <= 10 times: 8531\n",
      "Top 25 words:\n",
      "              Phrase  Characters  Terms  isStopword      Stem  Count  \\\n",
      "s                  s           1      1        True         s   2211   \n",
      "twitter      twitter           7      1       False   twitter   1423   \n",
      "com              com           3      1       False       com   1419   \n",
      "pic              pic           3      1       False       pic   1261   \n",
      "president  president           9      1       False    presid   1099   \n",
      "trump          trump           5      1       False     trump    993   \n",
      "@                  @           1      1       False         @    813   \n",
      "#                  #           1      1       False         #    777   \n",
      "t                  t           1      1        True         t    725   \n",
      "biden          biden           5      1       False     biden    573   \n",
      "donald        donald           6      1       False    donald    555   \n",
      "need            need           4      1       False      need    480   \n",
      "country      country           7      1       False   countri    466   \n",
      "vp                vp           2      1       False        vp    460   \n",
      "today          today           5      1       False     today    444   \n",
      "nation        nation           6      1       False    nation    418   \n",
      "m                  m           1      1        True         m    411   \n",
      "people        people           6      1       False     peopl    408   \n",
      "american    american           8      1       False  american    393   \n",
      "america      america           7      1       False   america    380   \n",
      "ll                ll           2      1        True        ll    364   \n",
      "make            make           4      1       False      make    361   \n",
      "day              day           3      1       False       day    359   \n",
      "time            time           4      1       False      time    339   \n",
      "ve                ve           2      1        True        ve    338   \n",
      "\n",
      "           Count Pct  Docs  Docs Pct  \n",
      "s           0.027864  1705  0.363230  \n",
      "twitter     0.017933  1422  0.302940  \n",
      "com         0.017883  1418  0.302088  \n",
      "pic         0.015891  1261  0.268641  \n",
      "president   0.013850   994  0.211760  \n",
      "trump       0.012514   928  0.197699  \n",
      "@           0.010246   664  0.141457  \n",
      "#           0.009792   728  0.155092  \n",
      "t           0.009137   645  0.137409  \n",
      "biden       0.007221   559  0.119088  \n",
      "donald      0.006994   549  0.116958  \n",
      "need        0.006049   437  0.093098  \n",
      "country     0.005873   453  0.096506  \n",
      "vp          0.005797   451  0.096080  \n",
      "today       0.005595   434  0.092458  \n",
      "nation      0.005268   401  0.085428  \n",
      "m           0.005180   378  0.080528  \n",
      "people      0.005142   382  0.081380  \n",
      "american    0.004953   375  0.079889  \n",
      "america     0.004789   370  0.078824  \n",
      "ll          0.004587   347  0.073924  \n",
      "make        0.004549   338  0.072007  \n",
      "day         0.004524   336  0.071581  \n",
      "time        0.004272   320  0.068172  \n",
      "ve          0.004260   310  0.066042  \n",
      "\n",
      "Bottom 25 words:\n",
      "                  Phrase  Characters  Terms  isStopword        Stem  Count  \\\n",
      "nn14                nn14           4      1       False        nn14      1   \n",
      "nmvlgzfkyn    nmvlgzfkyn          10      1       False  nmvlgzfkyn      1   \n",
      "nmftxeuy0o    nmftxeuy0o          10      1       False  nmftxeuy0o      1   \n",
      "disagree        disagree           8      1       False     disagre      1   \n",
      "dirigir          dirigir           7      1       False     dirigir      1   \n",
      "nrn4sc0m        nrn4sc0m           8      1       False    nrn4sc0m      1   \n",
      "dirigimos      dirigimos           9      1       False    dirigimo      1   \n",
      "nrdc                nrdc           4      1       False        nrdc      1   \n",
      "diploma          diploma           7      1       False     diploma      1   \n",
      "nr6yuklmry    nr6yuklmry          10      1       False  nr6yuklmri      1   \n",
      "nr1u0vovog    nr1u0vovog          10      1       False  nr1u0vovog      1   \n",
      "nql5ygnp        nql5ygnp           8      1       False    nql5ygnp      1   \n",
      "npy4jtug        npy4jtug           8      1       False    npy4jtug      1   \n",
      "nowthisnews  nowthisnews          11      1       False  nowthisnew      1   \n",
      "nowpic            nowpic           6      1       False      nowpic      1   \n",
      "diplomacia    diplomacia          10      1       False  diplomacia      1   \n",
      "nov                  nov           3      1       False         nov      1   \n",
      "notre              notre           5      1       False        notr      1   \n",
      "notified        notified           8      1       False      notifi      1   \n",
      "dir                  dir           3      1       False         dir      1   \n",
      "notch              notch           5      1       False       notch      1   \n",
      "notable          notable           7      1       False      notabl      1   \n",
      "nosed              nosed           5      1       False        nose      1   \n",
      "directorate  directorate          11      1       False    director      1   \n",
      "zzmlhu3cqi    zzmlhu3cqi          10      1       False  zzmlhu3cqi      1   \n",
      "\n",
      "             Count Pct  Docs  Docs Pct  \n",
      "nn14          0.000013     1  0.000213  \n",
      "nmvlgzfkyn    0.000013     1  0.000213  \n",
      "nmftxeuy0o    0.000013     1  0.000213  \n",
      "disagree      0.000013     1  0.000213  \n",
      "dirigir       0.000013     1  0.000213  \n",
      "nrn4sc0m      0.000013     1  0.000213  \n",
      "dirigimos     0.000013     1  0.000213  \n",
      "nrdc          0.000013     1  0.000213  \n",
      "diploma       0.000013     1  0.000213  \n",
      "nr6yuklmry    0.000013     1  0.000213  \n",
      "nr1u0vovog    0.000013     1  0.000213  \n",
      "nql5ygnp      0.000013     1  0.000213  \n",
      "npy4jtug      0.000013     1  0.000213  \n",
      "nowthisnews   0.000013     1  0.000213  \n",
      "nowpic        0.000013     1  0.000213  \n",
      "diplomacia    0.000013     1  0.000213  \n",
      "nov           0.000013     1  0.000213  \n",
      "notre         0.000013     1  0.000213  \n",
      "notified      0.000013     1  0.000213  \n",
      "dir           0.000013     1  0.000213  \n",
      "notch         0.000013     1  0.000213  \n",
      "notable       0.000013     1  0.000213  \n",
      "nosed         0.000013     1  0.000213  \n",
      "directorate   0.000013     1  0.000213  \n",
      "zzmlhu3cqi    0.000013     1  0.000213  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 4694\n",
      "Number of word forms (terms): 1877\n",
      "Number of words (tokens): 80671\n",
      "Mean words per document: 17.2\n",
      "Mean term occurance: 43.0\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 164\n",
      "Top 25 words:\n",
      "                          Phrase  Characters  Terms  isStopword  \\\n",
      "s                              s           1      1        True   \n",
      "twitter                  twitter           7      1       False   \n",
      "com                          com           3      1       False   \n",
      "twitter com          twitter com          11      2       False   \n",
      "pic twitter com  pic twitter com          15      3       False   \n",
      "pic twitter          pic twitter          11      2       False   \n",
      "pic                          pic           3      1       False   \n",
      "president              president           9      1       False   \n",
      "trump                      trump           5      1       False   \n",
      "@                              @           1      1       False   \n",
      "#                              #           1      1       False   \n",
      "t                              t           1      1        True   \n",
      "biden                      biden           5      1       False   \n",
      "donald                    donald           6      1       False   \n",
      "donald trump        donald trump          12      2       False   \n",
      "need                        need           4      1       False   \n",
      "country                  country           7      1       False   \n",
      "vp                            vp           2      1       False   \n",
      "today                      today           5      1       False   \n",
      "nation                    nation           6      1       False   \n",
      "m                              m           1      1        True   \n",
      "people                    people           6      1       False   \n",
      "american                american           8      1       False   \n",
      "america                  america           7      1       False   \n",
      "ll                            ll           2      1        True   \n",
      "\n",
      "                            Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "s                              s   2211   0.027408  1705  0.363230  \n",
      "twitter                  twitter   1423   0.017640  1422  0.302940  \n",
      "com                          com   1419   0.017590  1418  0.302088  \n",
      "twitter com          twitter com   1418   0.017578  1418  0.302088  \n",
      "pic twitter com  pic twitter com   1261   0.015631  1261  0.268641  \n",
      "pic twitter            pic twitt   1261   0.015631  1261  0.268641  \n",
      "pic                          pic   1261   0.015631  1261  0.268641  \n",
      "president                 presid   1099   0.013623   994  0.211760  \n",
      "trump                      trump    993   0.012309   928  0.197699  \n",
      "@                              @    813   0.010078   664  0.141457  \n",
      "#                              #    777   0.009632   728  0.155092  \n",
      "t                              t    725   0.008987   645  0.137409  \n",
      "biden                      biden    573   0.007103   559  0.119088  \n",
      "donald                    donald    555   0.006880   549  0.116958  \n",
      "donald trump        donald trump    555   0.006880   549  0.116958  \n",
      "need                        need    480   0.005950   437  0.093098  \n",
      "country                  countri    466   0.005777   453  0.096506  \n",
      "vp                            vp    460   0.005702   451  0.096080  \n",
      "today                      today    444   0.005504   434  0.092458  \n",
      "nation                    nation    418   0.005182   401  0.085428  \n",
      "m                              m    411   0.005095   378  0.080528  \n",
      "people                     peopl    408   0.005058   382  0.081380  \n",
      "american                american    393   0.004872   375  0.079889  \n",
      "america                  america    380   0.004710   370  0.078824  \n",
      "ll                            ll    364   0.004512   347  0.073924  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                          Phrase  Characters  Terms  \\\n",
      "president united states  president united states          23      3   \n",
      "president united                president united          16      2   \n",
      "dark                                        dark           4      1   \n",
      "began                                      began           5      1   \n",
      "day #                                      day #           5      2   \n",
      "beaten                                    beaten           6      1   \n",
      "president joe                      president joe          13      2   \n",
      "day president                      day president          13      2   \n",
      "timkaine                                timkaine           8      1   \n",
      "decency                                  decency           7      1   \n",
      "deep                                        deep           4      1   \n",
      "plan protect build            plan protect build          18      3   \n",
      "donald trump failed          donald trump failed          19      3   \n",
      "plan protect                        plan protect          12      2   \n",
      "auto industry                      auto industry          13      2   \n",
      "pete                                        pete           4      1   \n",
      "devos                                      devos           5      1   \n",
      "disaster                                disaster           8      1   \n",
      "pathway                                  pathway           7      1   \n",
      "disgrace                                disgrace           8      1   \n",
      "ask                                          ask           3      1   \n",
      "answer                                    answer           6      1   \n",
      "parents income                    parents income          14      2   \n",
      "trump needs                          trump needs          11      2   \n",
      "head rsvp                              head rsvp           9      2   \n",
      "\n",
      "                         isStopword                 Stem  Count  Count Pct  \\\n",
      "president united states       False  president united st     10   0.000124   \n",
      "president united              False       president unit     10   0.000124   \n",
      "dark                          False                 dark     10   0.000124   \n",
      "began                         False                began     10   0.000124   \n",
      "day #                         False                day #     10   0.000124   \n",
      "beaten                        False               beaten     10   0.000124   \n",
      "president joe                 False         president jo     10   0.000124   \n",
      "day president                 False           day presid     10   0.000124   \n",
      "timkaine                      False              timkain     10   0.000124   \n",
      "decency                       False               decenc     10   0.000124   \n",
      "deep                          False                 deep     10   0.000124   \n",
      "plan protect build            False   plan protect build     10   0.000124   \n",
      "donald trump failed           False    donald trump fail     10   0.000124   \n",
      "plan protect                  False         plan protect     10   0.000124   \n",
      "auto industry                 False        auto industri     10   0.000124   \n",
      "pete                          False                 pete     10   0.000124   \n",
      "devos                         False                 devo     10   0.000124   \n",
      "disaster                      False               disast     10   0.000124   \n",
      "pathway                       False              pathway     10   0.000124   \n",
      "disgrace                      False              disgrac     10   0.000124   \n",
      "ask                           False                  ask     10   0.000124   \n",
      "answer                        False               answer     10   0.000124   \n",
      "parents income                False        parents incom     10   0.000124   \n",
      "trump needs                   False             trump ne     10   0.000124   \n",
      "head rsvp                     False            head rsvp     10   0.000124   \n",
      "\n",
      "                         Docs  Docs Pct  \n",
      "president united states    10   0.00213  \n",
      "president united           10   0.00213  \n",
      "dark                       10   0.00213  \n",
      "began                      10   0.00213  \n",
      "day #                      10   0.00213  \n",
      "beaten                     10   0.00213  \n",
      "president joe              10   0.00213  \n",
      "day president              10   0.00213  \n",
      "timkaine                   10   0.00213  \n",
      "decency                    10   0.00213  \n",
      "deep                       10   0.00213  \n",
      "plan protect build         10   0.00213  \n",
      "donald trump failed        10   0.00213  \n",
      "plan protect               10   0.00213  \n",
      "auto industry              10   0.00213  \n",
      "pete                       10   0.00213  \n",
      "devos                      10   0.00213  \n",
      "disaster                   10   0.00213  \n",
      "pathway                    10   0.00213  \n",
      "disgrace                   10   0.00213  \n",
      "ask                        10   0.00213  \n",
      "answer                     10   0.00213  \n",
      "parents income             10   0.00213  \n",
      "trump needs                10   0.00213  \n",
      "head rsvp                  10   0.00213  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"JoeBidenTweets\", id_col=\"id\", text_col=\"tweet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 4999\n",
      "Number of word forms (terms): 14621\n",
      "Number of words (tokens): 221753\n",
      "Mean words per document: 44.4\n",
      "Mean term occurance: 15.2\n",
      "Number of terms occuring once: 6524\n",
      "Number of terms occuring <= 5 times: 10764\n",
      "Number of terms occuring <= 10 times: 12067\n",
      "Top 25 words:\n",
      "              Phrase  Characters  Terms  isStopword     Stem  Count  \\\n",
      "t                  t           1      1        True        t   3207   \n",
      "s                  s           1      1        True        s   3199   \n",
      "like            like           4      1       False     like   3057   \n",
      "good            good           4      1       False     good   2441   \n",
      "taste          taste           5      1       False     tast   2326   \n",
      "flavor        flavor           6      1       False   flavor   2085   \n",
      "coffee        coffee           6      1       False    coffe   1913   \n",
      "just            just           4      1        True     just   1854   \n",
      "tea              tea           3      1       False      tea   1491   \n",
      "great          great           5      1       False    great   1463   \n",
      "really        really           6      1       False   realli   1301   \n",
      "product      product           7      1       False  product   1276   \n",
      "love            love           4      1       False     love   1097   \n",
      "little        little           6      1       False    littl   1011   \n",
      "use              use           3      1       False      use    995   \n",
      "don              don           3      1        True      don    978   \n",
      "chocolate  chocolate           9      1       False   chocol    953   \n",
      "sugar          sugar           5      1       False    sugar    913   \n",
      "make            make           4      1       False     make    895   \n",
      "sweet          sweet           5      1       False    sweet    847   \n",
      "water          water           5      1       False    water    840   \n",
      "m                  m           1      1        True        m    817   \n",
      "cup              cup           3      1       False      cup    810   \n",
      "ve                ve           2      1        True       ve    777   \n",
      "better        better           6      1       False   better    768   \n",
      "\n",
      "           Count Pct  Docs  Docs Pct  \n",
      "t           0.014462  1882  0.376475  \n",
      "s           0.014426  1723  0.344669  \n",
      "like        0.013786  1918  0.383677  \n",
      "good        0.011008  1776  0.355271  \n",
      "taste       0.010489  1627  0.325465  \n",
      "flavor      0.009402  1429  0.285857  \n",
      "coffee      0.008627   732  0.146429  \n",
      "just        0.008361  1383  0.276655  \n",
      "tea         0.006724   506  0.101220  \n",
      "great       0.006597  1198  0.239648  \n",
      "really      0.005867   985  0.197039  \n",
      "product     0.005754   906  0.181236  \n",
      "love        0.004947   868  0.173635  \n",
      "little      0.004559   791  0.158232  \n",
      "use         0.004487   761  0.152230  \n",
      "don         0.004410   790  0.158032  \n",
      "chocolate   0.004298   463  0.092619  \n",
      "sugar       0.004117   577  0.115423  \n",
      "make        0.004036   743  0.148630  \n",
      "sweet       0.003820   626  0.125225  \n",
      "water       0.003788   483  0.096619  \n",
      "m           0.003684   609  0.121824  \n",
      "cup         0.003653   476  0.095219  \n",
      "ve          0.003504   622  0.124425  \n",
      "better      0.003463   657  0.131426  \n",
      "\n",
      "Bottom 25 words:\n",
      "                        Phrase  Characters  Terms  isStopword            Stem  \\\n",
      "overlooked          overlooked          10      1       False        overlook   \n",
      "overload              overload           8      1       False        overload   \n",
      "overlapping        overlapping          11      1       False         overlap   \n",
      "overkill              overkill           8      1       False         overkil   \n",
      "overjoyed            overjoyed           9      1       False         overjoy   \n",
      "overides              overides           8      1       False          overid   \n",
      "overheated          overheated          10      1       False           overh   \n",
      "overflowing        overflowing          11      1       False        overflow   \n",
      "overflowed          overflowed          10      1       False        overflow   \n",
      "overestimate      overestimate          12      1       False       overestim   \n",
      "overeating          overeating          10      1       False            over   \n",
      "overdrive            overdrive           9      1       False          overdr   \n",
      "overdosing          overdosing          10      1       False         overdos   \n",
      "overdo                  overdo           6      1       False          overdo   \n",
      "overcharged        overcharged          11      1       False       overcharg   \n",
      "overcame              overcame           8      1       False         overcam   \n",
      "cyanocobalomin  cyanocobalomin          14      1       False  cyanocobalomin   \n",
      "overbearingly    overbearingly          13      1       False   overbearingli   \n",
      "cycling                cycling           7      1       False            cycl   \n",
      "overally              overally           8      1       False            over   \n",
      "cyclists              cyclists           8      1       False         cyclist   \n",
      "overabundance    overabundance          13      1       False       overabund   \n",
      "cynicism              cynicism           8      1       False           cynic   \n",
      "d110                      d110           4      1       False            d110   \n",
      "kroger                  kroger           6      1       False          kroger   \n",
      "\n",
      "                Count  Count Pct  Docs  Docs Pct  \n",
      "overlooked          1   0.000005     1    0.0002  \n",
      "overload            1   0.000005     1    0.0002  \n",
      "overlapping         1   0.000005     1    0.0002  \n",
      "overkill            1   0.000005     1    0.0002  \n",
      "overjoyed           1   0.000005     1    0.0002  \n",
      "overides            1   0.000005     1    0.0002  \n",
      "overheated          1   0.000005     1    0.0002  \n",
      "overflowing         1   0.000005     1    0.0002  \n",
      "overflowed          1   0.000005     1    0.0002  \n",
      "overestimate        1   0.000005     1    0.0002  \n",
      "overeating          1   0.000005     1    0.0002  \n",
      "overdrive           1   0.000005     1    0.0002  \n",
      "overdosing          1   0.000005     1    0.0002  \n",
      "overdo              1   0.000005     1    0.0002  \n",
      "overcharged         1   0.000005     1    0.0002  \n",
      "overcame            1   0.000005     1    0.0002  \n",
      "cyanocobalomin      1   0.000005     1    0.0002  \n",
      "overbearingly       1   0.000005     1    0.0002  \n",
      "cycling             1   0.000005     1    0.0002  \n",
      "overally            1   0.000005     1    0.0002  \n",
      "cyclists            1   0.000005     1    0.0002  \n",
      "overabundance       1   0.000005     1    0.0002  \n",
      "cynicism            1   0.000005     1    0.0002  \n",
      "d110                1   0.000005     1    0.0002  \n",
      "kroger              1   0.000005     1    0.0002  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 4999\n",
      "Number of word forms (terms): 3656\n",
      "Number of words (tokens): 220067\n",
      "Mean words per document: 44.0\n",
      "Mean term occurance: 60.2\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 202\n",
      "Top 25 words:\n",
      "              Phrase  Characters  Terms  isStopword     Stem  Count  \\\n",
      "t                  t           1      1        True        t   3207   \n",
      "s                  s           1      1        True        s   3199   \n",
      "like            like           4      1       False     like   3057   \n",
      "good            good           4      1       False     good   2441   \n",
      "taste          taste           5      1       False     tast   2326   \n",
      "flavor        flavor           6      1       False   flavor   2085   \n",
      "coffee        coffee           6      1       False    coffe   1913   \n",
      "just            just           4      1        True     just   1854   \n",
      "tea              tea           3      1       False      tea   1491   \n",
      "great          great           5      1       False    great   1463   \n",
      "really        really           6      1       False   realli   1301   \n",
      "product      product           7      1       False  product   1276   \n",
      "love            love           4      1       False     love   1097   \n",
      "little        little           6      1       False    littl   1011   \n",
      "use              use           3      1       False      use    995   \n",
      "don              don           3      1        True      don    978   \n",
      "don t          don t           5      2       False    don t    969   \n",
      "chocolate  chocolate           9      1       False   chocol    953   \n",
      "sugar          sugar           5      1       False    sugar    913   \n",
      "make            make           4      1       False     make    895   \n",
      "sweet          sweet           5      1       False    sweet    847   \n",
      "water          water           5      1       False    water    840   \n",
      "m                  m           1      1        True        m    817   \n",
      "cup              cup           3      1       False      cup    810   \n",
      "ve                ve           2      1        True       ve    777   \n",
      "\n",
      "           Count Pct  Docs  Docs Pct  \n",
      "t           0.014573  1882  0.376475  \n",
      "s           0.014536  1723  0.344669  \n",
      "like        0.013891  1918  0.383677  \n",
      "good        0.011092  1776  0.355271  \n",
      "taste       0.010570  1627  0.325465  \n",
      "flavor      0.009474  1429  0.285857  \n",
      "coffee      0.008693   732  0.146429  \n",
      "just        0.008425  1383  0.276655  \n",
      "tea         0.006775   506  0.101220  \n",
      "great       0.006648  1198  0.239648  \n",
      "really      0.005912   985  0.197039  \n",
      "product     0.005798   906  0.181236  \n",
      "love        0.004985   868  0.173635  \n",
      "little      0.004594   791  0.158232  \n",
      "use         0.004521   761  0.152230  \n",
      "don         0.004444   790  0.158032  \n",
      "don t       0.004403   784  0.156831  \n",
      "chocolate   0.004330   463  0.092619  \n",
      "sugar       0.004149   577  0.115423  \n",
      "make        0.004067   743  0.148630  \n",
      "sweet       0.003849   626  0.125225  \n",
      "water       0.003817   483  0.096619  \n",
      "m           0.003713   609  0.121824  \n",
      "cup         0.003681   476  0.095219  \n",
      "ve          0.003531   622  0.124425  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                  Phrase  Characters  Terms  isStopword  \\\n",
      "like fruit                    like fruit          10      2       False   \n",
      "t purchase                    t purchase          10      2       False   \n",
      "far better                    far better          10      2       False   \n",
      "satisfies                      satisfies           9      1       False   \n",
      "like said                      like said           9      2       False   \n",
      "explain                          explain           7      1       False   \n",
      "satisfy sweet tooth  satisfy sweet tooth          19      3       False   \n",
      "like think                    like think          10      2       False   \n",
      "excuse                            excuse           6      1       False   \n",
      "ve eating                      ve eating           9      2       False   \n",
      "think better                think better          12      2       False   \n",
      "sauce just                    sauce just          10      2       False   \n",
      "lipton tea                    lipton tea          10      2       False   \n",
      "enjoy taste                  enjoy taste          11      2       False   \n",
      "em                                    em           2      1       False   \n",
      "elbow                              elbow           5      1       False   \n",
      "saving                            saving           6      1       False   \n",
      "earlier                          earlier           7      1       False   \n",
      "little hard                  little hard          11      2       False   \n",
      "little pricey              little pricey          13      2       False   \n",
      "drink black                  drink black          11      2       False   \n",
      "drier                              drier           5      1       False   \n",
      "drawer                            drawer           6      1       False   \n",
      "little snack                little snack          12      2       False   \n",
      "product nice                product nice          12      2       False   \n",
      "\n",
      "                                    Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "like fruit                    like fruit     10   0.000045    10     0.002  \n",
      "t purchase                     t purchas     10   0.000045    10     0.002  \n",
      "far better                      far bett     10   0.000045    10     0.002  \n",
      "satisfies                        satisfi     10   0.000045    10     0.002  \n",
      "like said                      like said     10   0.000045    10     0.002  \n",
      "explain                          explain     10   0.000045    10     0.002  \n",
      "satisfy sweet tooth  satisfy sweet tooth     10   0.000045    10     0.002  \n",
      "like think                    like think     10   0.000045    10     0.002  \n",
      "excuse                             excus     10   0.000045    10     0.002  \n",
      "ve eating                         ve eat     10   0.000045    10     0.002  \n",
      "think better                  think bett     10   0.000045    10     0.002  \n",
      "sauce just                    sauce just     10   0.000045    10     0.002  \n",
      "lipton tea                    lipton tea     10   0.000045    10     0.002  \n",
      "enjoy taste                   enjoy tast     10   0.000045    10     0.002  \n",
      "em                                    em     10   0.000045    10     0.002  \n",
      "elbow                              elbow     10   0.000045    10     0.002  \n",
      "saving                              save     10   0.000045    10     0.002  \n",
      "earlier                          earlier     10   0.000045    10     0.002  \n",
      "little hard                  little hard     10   0.000045    10     0.002  \n",
      "little pricey              little pricey     10   0.000045    10     0.002  \n",
      "drink black                  drink black     10   0.000045    10     0.002  \n",
      "drier                              drier     10   0.000045    10     0.002  \n",
      "drawer                            drawer     10   0.000045    10     0.002  \n",
      "little snack                little snack     10   0.000045    10     0.002  \n",
      "product nice                 product nic     10   0.000045    10     0.002  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"amazon_food_reviews_10\", id_col=\"reviewID\", text_col=\"reviewText\", target_cols=['overall'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 4637\n",
      "Number of word forms (terms): 8433\n",
      "Number of words (tokens): 89624\n",
      "Mean words per document: 19.3\n",
      "Mean term occurance: 10.6\n",
      "Number of terms occuring once: 4194\n",
      "Number of terms occuring <= 5 times: 6636\n",
      "Number of terms occuring <= 10 times: 7296\n",
      "Top 25 words:\n",
      "                Phrase  Characters  Terms  isStopword      Stem  Count  \\\n",
      "temple          temple           6      1       False     templ   4410   \n",
      "place            place           5      1       False     place   2636   \n",
      "krishna        krishna           7      1       False   krishna   1884   \n",
      "visit            visit           5      1       False     visit   1711   \n",
      "hare              hare           4      1       False      hare   1350   \n",
      "good              good           4      1       False      good   1195   \n",
      "bangalore    bangalore           9      1       False  bangalor   1024   \n",
      "iskcon          iskcon           6      1       False    iskcon    822   \n",
      "nice              nice           4      1       False      nice    800   \n",
      "beautiful    beautiful           9      1       False    beauti    779   \n",
      "food              food           4      1       False      food    756   \n",
      "lord              lord           4      1       False      lord    611   \n",
      "time              time           4      1       False      time    609   \n",
      "s                    s           1      1        True         s    607   \n",
      "like              like           4      1       False      like    544   \n",
      "clean            clean           5      1       False     clean    482   \n",
      "experience  experience          10      1       False    experi    481   \n",
      "inside          inside           6      1       False     insid    480   \n",
      "visited        visited           7      1       False     visit    461   \n",
      "peaceful      peaceful           8      1       False      peac    457   \n",
      "peace            peace           5      1       False      peac    449   \n",
      "maintained  maintained          10      1       False  maintain    446   \n",
      "rama              rama           4      1       False      rama    446   \n",
      "great            great           5      1       False     great    423   \n",
      "main              main           4      1       False      main    414   \n",
      "\n",
      "            Count Pct  Docs  Docs Pct  \n",
      "temple       0.049206  2499  0.538926  \n",
      "place        0.029412  1975  0.425922  \n",
      "krishna      0.021021  1197  0.258141  \n",
      "visit        0.019091  1426  0.307526  \n",
      "hare         0.015063   459  0.098986  \n",
      "good         0.013333   999  0.215441  \n",
      "bangalore    0.011426   886  0.191072  \n",
      "iskcon       0.009172   671  0.144706  \n",
      "nice         0.008926   710  0.153116  \n",
      "beautiful    0.008692   708  0.152685  \n",
      "food         0.008435   641  0.138236  \n",
      "lord         0.006817   487  0.105025  \n",
      "time         0.006795   514  0.110848  \n",
      "s            0.006773   469  0.101143  \n",
      "like         0.006070   463  0.099849  \n",
      "clean        0.005378   467  0.100712  \n",
      "experience   0.005367   426  0.091870  \n",
      "inside       0.005356   406  0.087557  \n",
      "visited      0.005144   419  0.090360  \n",
      "peaceful     0.005099   451  0.097261  \n",
      "peace        0.005010   421  0.090791  \n",
      "maintained   0.004976   437  0.094242  \n",
      "rama         0.004976   224  0.048307  \n",
      "great        0.004720   370  0.079793  \n",
      "main         0.004619   321  0.069226  \n",
      "\n",
      "Bottom 25 words:\n",
      "                        Phrase  Characters  Terms  isStopword           Stem  \\\n",
      "imagining            imagining           9      1       False         imagin   \n",
      "imaging                imaging           7      1       False           imag   \n",
      "imaginative        imaginative          11      1       False         imagin   \n",
      "incident              incident           8      1       False          incid   \n",
      "incite                  incite           6      1       False          incit   \n",
      "includding          includding          10      1       False         includ   \n",
      "inclusion            inclusion           9      1       False         inclus   \n",
      "inducing              inducing           8      1       False          induc   \n",
      "indrukwekkende  indrukwekkende          14      1       False  indrukwekkend   \n",
      "indou                    indou           5      1       False          indou   \n",
      "indoors                indoors           7      1       False         indoor   \n",
      "individually      individually          12      1       False       individu   \n",
      "indira                  indira           6      1       False         indira   \n",
      "indescribable    indescribable          13      1       False      indescrib   \n",
      "independent        independent          11      1       False       independ   \n",
      "indefinable        indefinable          11      1       False        indefin   \n",
      "ind                        ind           3      1       False            ind   \n",
      "increasing          increasing          10      1       False        increas   \n",
      "incorrect            incorrect           9      1       False      incorrect   \n",
      "incorporates      incorporates          12      1       False       incorpor   \n",
      "incomplte            incomplte           9      1       False       incomplt   \n",
      "incomparable      incomparable          12      1       False       incompar   \n",
      "inclusive            inclusive           9      1       False         inclus   \n",
      "inclusions          inclusions          10      1       False         inclus   \n",
      "zus                        zus           3      1       False             zu   \n",
      "\n",
      "                Count  Count Pct  Docs  Docs Pct  \n",
      "imagining           1   0.000011     1  0.000216  \n",
      "imaging             1   0.000011     1  0.000216  \n",
      "imaginative         1   0.000011     1  0.000216  \n",
      "incident            1   0.000011     1  0.000216  \n",
      "incite              1   0.000011     1  0.000216  \n",
      "includding          1   0.000011     1  0.000216  \n",
      "inclusion           1   0.000011     1  0.000216  \n",
      "inducing            1   0.000011     1  0.000216  \n",
      "indrukwekkende      1   0.000011     1  0.000216  \n",
      "indou               1   0.000011     1  0.000216  \n",
      "indoors             1   0.000011     1  0.000216  \n",
      "individually        1   0.000011     1  0.000216  \n",
      "indira              1   0.000011     1  0.000216  \n",
      "indescribable       1   0.000011     1  0.000216  \n",
      "independent         1   0.000011     1  0.000216  \n",
      "indefinable         1   0.000011     1  0.000216  \n",
      "ind                 1   0.000011     1  0.000216  \n",
      "increasing          1   0.000011     1  0.000216  \n",
      "incorrect           1   0.000011     1  0.000216  \n",
      "incorporates        1   0.000011     1  0.000216  \n",
      "incomplte           1   0.000011     1  0.000216  \n",
      "incomparable        1   0.000011     1  0.000216  \n",
      "inclusive           1   0.000011     1  0.000216  \n",
      "inclusions          1   0.000011     1  0.000216  \n",
      "zus                 1   0.000011     1  0.000216  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 4637\n",
      "Number of word forms (terms): 1883\n",
      "Number of words (tokens): 94187\n",
      "Mean words per document: 20.3\n",
      "Mean term occurance: 50.0\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 145\n",
      "Top 25 words:\n",
      "                    Phrase  Characters  Terms  isStopword          Stem  \\\n",
      "temple              temple           6      1       False         templ   \n",
      "place                place           5      1       False         place   \n",
      "krishna            krishna           7      1       False       krishna   \n",
      "visit                visit           5      1       False         visit   \n",
      "hare                  hare           4      1       False          hare   \n",
      "good                  good           4      1       False          good   \n",
      "bangalore        bangalore           9      1       False      bangalor   \n",
      "iskcon              iskcon           6      1       False        iskcon   \n",
      "nice                  nice           4      1       False          nice   \n",
      "beautiful        beautiful           9      1       False        beauti   \n",
      "food                  food           4      1       False          food   \n",
      "lord                  lord           4      1       False          lord   \n",
      "time                  time           4      1       False          time   \n",
      "s                        s           1      1        True             s   \n",
      "hare krishna  hare krishna          12      2       False  hare krishna   \n",
      "like                  like           4      1       False          like   \n",
      "clean                clean           5      1       False         clean   \n",
      "experience      experience          10      1       False        experi   \n",
      "inside              inside           6      1       False         insid   \n",
      "visited            visited           7      1       False         visit   \n",
      "peaceful          peaceful           8      1       False          peac   \n",
      "peace                peace           5      1       False          peac   \n",
      "rama                  rama           4      1       False          rama   \n",
      "maintained      maintained          10      1       False      maintain   \n",
      "great                great           5      1       False         great   \n",
      "\n",
      "              Count  Count Pct  Docs  Docs Pct  \n",
      "temple         4410   0.046822  2499  0.538926  \n",
      "place          2636   0.027987  1975  0.425922  \n",
      "krishna        1884   0.020003  1197  0.258141  \n",
      "visit          1711   0.018166  1426  0.307526  \n",
      "hare           1350   0.014333   459  0.098986  \n",
      "good           1195   0.012688   999  0.215441  \n",
      "bangalore      1024   0.010872   886  0.191072  \n",
      "iskcon          822   0.008727   671  0.144706  \n",
      "nice            800   0.008494   710  0.153116  \n",
      "beautiful       779   0.008271   708  0.152685  \n",
      "food            756   0.008027   641  0.138236  \n",
      "lord            611   0.006487   487  0.105025  \n",
      "time            609   0.006466   514  0.110848  \n",
      "s               607   0.006445   469  0.101143  \n",
      "hare krishna    555   0.005893   418  0.090144  \n",
      "like            544   0.005776   463  0.099849  \n",
      "clean           482   0.005117   467  0.100712  \n",
      "experience      481   0.005107   426  0.091870  \n",
      "inside          480   0.005096   406  0.087557  \n",
      "visited         461   0.004895   419  0.090360  \n",
      "peaceful        457   0.004852   451  0.097261  \n",
      "peace           449   0.004767   421  0.090791  \n",
      "rama            446   0.004735   224  0.048307  \n",
      "maintained      446   0.004735   437  0.094242  \n",
      "great           423   0.004491   370  0.079793  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                        Phrase  Characters  Terms  isStopword  \\\n",
      "scenic                                  scenic           6      1       False   \n",
      "security check                  security check          14      2       False   \n",
      "visit temple bangalore  visit temple bangalore          22      3       False   \n",
      "biggest iskcon                  biggest iskcon          14      2       False   \n",
      "marvel                                  marvel           6      1       False   \n",
      "founder                                founder           7      1       False   \n",
      "finest                                  finest           6      1       False   \n",
      "reach place                        reach place          11      2       False   \n",
      "books food                          books food          10      2       False   \n",
      "really feel                        really feel          11      2       False   \n",
      "temple hare krishna        temple hare krishna          19      3       False   \n",
      "moments                                moments           7      1       False   \n",
      "relax mind                          relax mind          10      2       False   \n",
      "conduct                                conduct           7      1       False   \n",
      "mins                                      mins           4      1       False   \n",
      "blessings lord krishna  blessings lord krishna          22      3       False   \n",
      "mental peace                      mental peace          12      2       False   \n",
      "ago                                        ago           3      1       False   \n",
      "control                                control           7      1       False   \n",
      "vedic                                    vedic           5      1       False   \n",
      "material                              material           8      1       False   \n",
      "temple enjoy                      temple enjoy          12      2       False   \n",
      "veg restaurant                  veg restaurant          14      2       False   \n",
      "finish                                  finish           6      1       False   \n",
      "maintained neat                maintained neat          15      2       False   \n",
      "\n",
      "                                          Stem  Count  Count Pct  Docs  \\\n",
      "scenic                                  scenic     10   0.000106    10   \n",
      "security check                  security check     10   0.000106    10   \n",
      "visit temple bangalore   visit temple bangalor     10   0.000106    10   \n",
      "biggest iskcon                  biggest iskcon     10   0.000106    10   \n",
      "marvel                                  marvel     10   0.000106    10   \n",
      "founder                                founder     10   0.000106    10   \n",
      "finest                                  finest     10   0.000106    10   \n",
      "reach place                         reach plac     10   0.000106    10   \n",
      "books food                          books food     10   0.000106    10   \n",
      "really feel                        really feel     10   0.000106    10   \n",
      "temple hare krishna        temple hare krishna     10   0.000106    10   \n",
      "moments                                 moment     10   0.000106    10   \n",
      "relax mind                          relax mind     10   0.000106    10   \n",
      "conduct                                conduct     10   0.000106    10   \n",
      "mins                                       min     10   0.000106    10   \n",
      "blessings lord krishna  blessings lord krishna     10   0.000106    10   \n",
      "mental peace                       mental peac     10   0.000106    10   \n",
      "ago                                        ago     10   0.000106    10   \n",
      "control                                control     10   0.000106    10   \n",
      "vedic                                    vedic     10   0.000106    10   \n",
      "material                                materi     10   0.000106    10   \n",
      "temple enjoy                      temple enjoy     10   0.000106    10   \n",
      "veg restaurant                     veg restaur     10   0.000106    10   \n",
      "finish                                  finish     10   0.000106    10   \n",
      "maintained neat                maintained neat     10   0.000106    10   \n",
      "\n",
      "                        Docs Pct  \n",
      "scenic                  0.002157  \n",
      "security check          0.002157  \n",
      "visit temple bangalore  0.002157  \n",
      "biggest iskcon          0.002157  \n",
      "marvel                  0.002157  \n",
      "founder                 0.002157  \n",
      "finest                  0.002157  \n",
      "reach place             0.002157  \n",
      "books food              0.002157  \n",
      "really feel             0.002157  \n",
      "temple hare krishna     0.002157  \n",
      "moments                 0.002157  \n",
      "relax mind              0.002157  \n",
      "conduct                 0.002157  \n",
      "mins                    0.002157  \n",
      "blessings lord krishna  0.002157  \n",
      "mental peace            0.002157  \n",
      "ago                     0.002157  \n",
      "control                 0.002157  \n",
      "vedic                   0.002157  \n",
      "material                0.002157  \n",
      "temple enjoy            0.002157  \n",
      "veg restaurant          0.002157  \n",
      "finish                  0.002157  \n",
      "maintained neat         0.002157  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"ISKON_IMB767-XLS-ENG\", id_col=\"ID\", text_col=\"text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 10000\n",
      "Number of word forms (terms): 50000\n",
      "Number of words (tokens): 1164195\n",
      "Mean words per document: 116.4\n",
      "Mean term occurance: 23.3\n",
      "Number of terms occuring once: 17580\n",
      "Number of terms occuring <= 5 times: 33642\n",
      "Number of terms occuring <= 10 times: 39047\n",
      "Top 25 words:\n",
      "                Phrase  Characters  Terms  isStopword     Stem  Count  \\\n",
      "br                  br           2      1       False       br  40210   \n",
      "s                    s           1      1        True        s  25425   \n",
      "movie            movie           5      1       False     movi  17448   \n",
      "film              film           4      1       False     film  16063   \n",
      "t                    t           1      1        True        t  13355   \n",
      "like              like           4      1       False     like   7993   \n",
      "just              just           4      1        True     just   6862   \n",
      "good              good           4      1       False     good   5925   \n",
      "time              time           4      1       False     time   5001   \n",
      "story            story           5      1       False    stori   4644   \n",
      "really          really           6      1       False   realli   4546   \n",
      "great            great           5      1       False    great   3697   \n",
      "bad                bad           3      1       False      bad   3583   \n",
      "people          people           6      1       False    peopl   3562   \n",
      "don                don           3      1        True      don   3411   \n",
      "make              make           4      1       False     make   3222   \n",
      "way                way           3      1       False      way   3174   \n",
      "movies          movies           6      1       False     movi   3076   \n",
      "characters  characters          10      1       False  charact   2885   \n",
      "watch            watch           5      1       False    watch   2854   \n",
      "films            films           5      1       False     film   2840   \n",
      "think            think           5      1       False    think   2798   \n",
      "plot              plot           4      1       False     plot   2675   \n",
      "character    character           9      1       False  charact   2661   \n",
      "seen              seen           4      1       False     seen   2628   \n",
      "\n",
      "            Count Pct  Docs  Docs Pct  \n",
      "br           0.034539  5761    0.5761  \n",
      "s            0.021839  7227    0.7227  \n",
      "movie        0.014987  6119    0.6119  \n",
      "film         0.013798  5597    0.5597  \n",
      "t            0.011471  5967    0.5967  \n",
      "like         0.006866  4608    0.4608  \n",
      "just         0.005894  4129    0.4129  \n",
      "good         0.005089  3764    0.3764  \n",
      "time         0.004296  3480    0.3480  \n",
      "story        0.003989  3019    0.3019  \n",
      "really       0.003905  2953    0.2953  \n",
      "great        0.003176  2542    0.2542  \n",
      "bad          0.003078  2347    0.2347  \n",
      "people       0.003060  2434    0.2434  \n",
      "don          0.002930  2547    0.2547  \n",
      "make         0.002768  2454    0.2454  \n",
      "way          0.002726  2346    0.2346  \n",
      "movies       0.002642  2119    0.2119  \n",
      "characters   0.002478  2035    0.2035  \n",
      "watch        0.002451  2189    0.2189  \n",
      "films        0.002439  1878    0.1878  \n",
      "think        0.002403  2138    0.2138  \n",
      "plot         0.002298  2017    0.2017  \n",
      "character    0.002286  1838    0.1838  \n",
      "seen         0.002257  2079    0.2079  \n",
      "\n",
      "Bottom 25 words:\n",
      "                      Phrase  Characters  Terms  isStopword           Stem  \\\n",
      "knitt                  knitt           5      1       False          knitt   \n",
      "kold                    kold           4      1       False           kold   \n",
      "knockabout        knockabout          10      1       False     knockabout   \n",
      "kolb                    kolb           4      1       False           kolb   \n",
      "kokoda                kokoda           6      1       False         kokoda   \n",
      "kokkinos            kokkinos           8      1       False        kokkino   \n",
      "kojak                  kojak           5      1       False          kojak   \n",
      "koichi                koichi           6      1       False         koichi   \n",
      "kohram                kohram           6      1       False         kohram   \n",
      "kohala                kohala           6      1       False         kohala   \n",
      "kodi                    kodi           4      1       False           kodi   \n",
      "kodak                  kodak           5      1       False          kodak   \n",
      "kodachrome        kodachrome          10      1       False      kodachrom   \n",
      "koboi                  koboi           5      1       False          koboi   \n",
      "kobe                    kobe           4      1       False           kobe   \n",
      "ko                        ko           2      1       False             ko   \n",
      "knucklehead      knucklehead          11      1       False    knucklehead   \n",
      "knuckled            knuckled           8      1       False         knuckl   \n",
      "knowledgable    knowledgable          12      1       False       knowledg   \n",
      "knowingly          knowingly           9      1       False      knowingli   \n",
      "knotty                knotty           6      1       False         knotti   \n",
      "knopfler            knopfler           8      1       False       knopfler   \n",
      "knoller              knoller           7      1       False        knoller   \n",
      "knockdown          knockdown           9      1       False      knockdown   \n",
      "zzzzzzzzzzzzz  zzzzzzzzzzzzz          13      1       False  zzzzzzzzzzzzz   \n",
      "\n",
      "               Count     Count Pct  Docs  Docs Pct  \n",
      "knitt              1  8.589626e-07     1    0.0001  \n",
      "kold               1  8.589626e-07     1    0.0001  \n",
      "knockabout         1  8.589626e-07     1    0.0001  \n",
      "kolb               1  8.589626e-07     1    0.0001  \n",
      "kokoda             1  8.589626e-07     1    0.0001  \n",
      "kokkinos           1  8.589626e-07     1    0.0001  \n",
      "kojak              1  8.589626e-07     1    0.0001  \n",
      "koichi             1  8.589626e-07     1    0.0001  \n",
      "kohram             1  8.589626e-07     1    0.0001  \n",
      "kohala             1  8.589626e-07     1    0.0001  \n",
      "kodi               1  8.589626e-07     1    0.0001  \n",
      "kodak              1  8.589626e-07     1    0.0001  \n",
      "kodachrome         1  8.589626e-07     1    0.0001  \n",
      "koboi              1  8.589626e-07     1    0.0001  \n",
      "kobe               1  8.589626e-07     1    0.0001  \n",
      "ko                 1  8.589626e-07     1    0.0001  \n",
      "knucklehead        1  8.589626e-07     1    0.0001  \n",
      "knuckled           1  8.589626e-07     1    0.0001  \n",
      "knowledgable       1  8.589626e-07     1    0.0001  \n",
      "knowingly          1  8.589626e-07     1    0.0001  \n",
      "knotty             1  8.589626e-07     1    0.0001  \n",
      "knopfler           1  8.589626e-07     1    0.0001  \n",
      "knoller            1  8.589626e-07     1    0.0001  \n",
      "knockdown          1  8.589626e-07     1    0.0001  \n",
      "zzzzzzzzzzzzz      1  8.589626e-07     1    0.0001  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 10000\n",
      "Number of word forms (terms): 18178\n",
      "Number of words (tokens): 1278500\n",
      "Mean words per document: 127.8\n",
      "Mean term occurance: 70.3\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 1191\n",
      "Top 25 words:\n",
      "                Phrase  Characters  Terms  isStopword     Stem  Count  \\\n",
      "br                  br           2      1       False       br  40210   \n",
      "s                    s           1      1        True        s  25425   \n",
      "br br            br br           5      2       False    br br  20134   \n",
      "movie            movie           5      1       False     movi  17448   \n",
      "film              film           4      1       False     film  16063   \n",
      "t                    t           1      1        True        t  13355   \n",
      "like              like           4      1       False     like   7993   \n",
      "just              just           4      1        True     just   6862   \n",
      "good              good           4      1       False     good   5925   \n",
      "time              time           4      1       False     time   5001   \n",
      "story            story           5      1       False    stori   4644   \n",
      "really          really           6      1       False   realli   4546   \n",
      "great            great           5      1       False    great   3697   \n",
      "bad                bad           3      1       False      bad   3583   \n",
      "people          people           6      1       False    peopl   3562   \n",
      "don                don           3      1        True      don   3411   \n",
      "don t            don t           5      2       False    don t   3300   \n",
      "make              make           4      1       False     make   3222   \n",
      "way                way           3      1       False      way   3174   \n",
      "movies          movies           6      1       False     movi   3076   \n",
      "characters  characters          10      1       False  charact   2885   \n",
      "watch            watch           5      1       False    watch   2854   \n",
      "films            films           5      1       False     film   2840   \n",
      "think            think           5      1       False    think   2798   \n",
      "plot              plot           4      1       False     plot   2675   \n",
      "\n",
      "            Count Pct  Docs  Docs Pct  \n",
      "br           0.031451  5761    0.5761  \n",
      "s            0.019887  7227    0.7227  \n",
      "br br        0.015748  5761    0.5761  \n",
      "movie        0.013647  6119    0.6119  \n",
      "film         0.012564  5597    0.5597  \n",
      "t            0.010446  5967    0.5967  \n",
      "like         0.006252  4608    0.4608  \n",
      "just         0.005367  4129    0.4129  \n",
      "good         0.004634  3764    0.3764  \n",
      "time         0.003912  3480    0.3480  \n",
      "story        0.003632  3019    0.3019  \n",
      "really       0.003556  2953    0.2953  \n",
      "great        0.002892  2542    0.2542  \n",
      "bad          0.002803  2347    0.2347  \n",
      "people       0.002786  2434    0.2434  \n",
      "don          0.002668  2547    0.2547  \n",
      "don t        0.002581  2478    0.2478  \n",
      "make         0.002520  2454    0.2454  \n",
      "way          0.002483  2346    0.2346  \n",
      "movies       0.002406  2119    0.2119  \n",
      "characters   0.002257  2035    0.2035  \n",
      "watch        0.002232  2189    0.2189  \n",
      "films        0.002221  1878    0.1878  \n",
      "think        0.002189  2138    0.2138  \n",
      "plot         0.002092  2017    0.2017  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                Phrase  Characters  Terms  isStopword  \\\n",
      "blasting                      blasting           8      1       False   \n",
      "movie television      movie television          16      2       False   \n",
      "t watching                  t watching          10      2       False   \n",
      "movie tell                  movie tell          10      2       False   \n",
      "t warn                          t warn           6      2       False   \n",
      "t walk                          t walk           6      2       False   \n",
      "movie title                movie title          11      2       False   \n",
      "t understand movie  t understand movie          18      3       False   \n",
      "blessing                      blessing           8      1       False   \n",
      "blindly                        blindly           7      1       False   \n",
      "t touch                        t touch           7      2       False   \n",
      "t totally                    t totally           9      2       False   \n",
      "bliss                            bliss           5      1       False   \n",
      "t things                      t things           8      2       False   \n",
      "t stop laughing        t stop laughing          15      3       False   \n",
      "fiddle                          fiddle           6      1       False   \n",
      "fiction movie            fiction movie          13      2       False   \n",
      "t simply                      t simply           8      2       False   \n",
      "t said                          t said           6      2       False   \n",
      "blurb                            blurb           5      1       False   \n",
      "t run                            t run           5      2       False   \n",
      "t ring                          t ring           6      2       False   \n",
      "t resist                      t resist           8      2       False   \n",
      "blurry                          blurry           6      1       False   \n",
      "imdb rating                imdb rating          11      2       False   \n",
      "\n",
      "                                 Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "blasting                        blast     10   0.000008    10     0.001  \n",
      "movie television        movie televis     10   0.000008    10     0.001  \n",
      "t watching                    t watch     10   0.000008    10     0.001  \n",
      "movie tell                  movie tel     10   0.000008    10     0.001  \n",
      "t warn                         t warn     10   0.000008    10     0.001  \n",
      "t walk                         t walk     10   0.000008    10     0.001  \n",
      "movie title                movie titl     10   0.000008    10     0.001  \n",
      "t understand movie  t understand movi     10   0.000008    10     0.001  \n",
      "blessing                        bless     10   0.000008    10     0.001  \n",
      "blindly                       blindli     10   0.000008    10     0.001  \n",
      "t touch                       t touch     10   0.000008    10     0.001  \n",
      "t totally                     t total     10   0.000008    10     0.001  \n",
      "bliss                           bliss     10   0.000008    10     0.001  \n",
      "t things                      t thing     10   0.000008    10     0.001  \n",
      "t stop laughing          t stop laugh     10   0.000008    10     0.001  \n",
      "fiddle                          fiddl     10   0.000008    10     0.001  \n",
      "fiction movie            fiction movi     10   0.000008    10     0.001  \n",
      "t simply                     t simpli     10   0.000008    10     0.001  \n",
      "t said                         t said     10   0.000008    10     0.001  \n",
      "blurb                           blurb     10   0.000008    10     0.001  \n",
      "t run                           t run     10   0.000008    10     0.001  \n",
      "t ring                         t ring     10   0.000008    10     0.001  \n",
      "t resist                     t resist     10   0.000008    10     0.001  \n",
      "blurry                         blurri     10   0.000008    10     0.001  \n",
      "imdb rating                  imdb rat     10   0.000008    10     0.001  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"imdb.small\", id_col=\"id\", text_col=\"en\", target_cols=['rating', 'score'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n",
      "Number of documents: 49994\n",
      "Number of word forms (terms): 44433\n",
      "Number of words (tokens): nan\n",
      "Mean words per document: nan\n",
      "Mean term occurance: nan\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 0\n",
      "Top 25 words:\n",
      "          Phrase  Characters  Terms  isStopword     Stem  Count  Count Pct  \\\n",
      "#              #           1      1       False        #    NaN        NaN   \n",
      "#1            #1           2      1       False       #1    NaN        NaN   \n",
      "#10          #10           3      1       False      #10    NaN        NaN   \n",
      "#10003    #10003           6      1       False   #10003    NaN        NaN   \n",
      "#10084    #10084           6      1       False   #10084    NaN        NaN   \n",
      "#1076      #1076           5      1       False    #1076    NaN        NaN   \n",
      "#1082      #1082           5      1       False    #1082    NaN        NaN   \n",
      "#1086      #1086           5      1       False    #1086    NaN        NaN   \n",
      "#1087      #1087           5      1       False    #1087    NaN        NaN   \n",
      "#1088      #1088           5      1       False    #1088    NaN        NaN   \n",
      "#1091      #1091           5      1       False    #1091    NaN        NaN   \n",
      "#11          #11           3      1       False      #11    NaN        NaN   \n",
      "#1150      #1150           5      1       False    #1150    NaN        NaN   \n",
      "#12290    #12290           6      1       False   #12290    NaN        NaN   \n",
      "#12356    #12356           6      1       False   #12356    NaN        NaN   \n",
      "#12424    #12424           6      1       False   #12424    NaN        NaN   \n",
      "#128522  #128522           7      1       False  #128522    NaN        NaN   \n",
      "#128523  #128523           7      1       False  #128523    NaN        NaN   \n",
      "#128541  #128541           7      1       False  #128541    NaN        NaN   \n",
      "#2            #2           2      1       False       #2    NaN        NaN   \n",
      "#21619    #21619           6      1       False   #21619    NaN        NaN   \n",
      "#2951      #2951           5      1       False    #2951    NaN        NaN   \n",
      "#2for      #2for           5      1       False    #2for    NaN        NaN   \n",
      "#3            #3           2      1       False       #3    NaN        NaN   \n",
      "#30          #30           3      1       False      #30    NaN        NaN   \n",
      "\n",
      "         Docs  Docs Pct  \n",
      "#         NaN       NaN  \n",
      "#1        NaN       NaN  \n",
      "#10       NaN       NaN  \n",
      "#10003    NaN       NaN  \n",
      "#10084    NaN       NaN  \n",
      "#1076     NaN       NaN  \n",
      "#1082     NaN       NaN  \n",
      "#1086     NaN       NaN  \n",
      "#1087     NaN       NaN  \n",
      "#1088     NaN       NaN  \n",
      "#1091     NaN       NaN  \n",
      "#11       NaN       NaN  \n",
      "#1150     NaN       NaN  \n",
      "#12290    NaN       NaN  \n",
      "#12356    NaN       NaN  \n",
      "#12424    NaN       NaN  \n",
      "#128522   NaN       NaN  \n",
      "#128523   NaN       NaN  \n",
      "#128541   NaN       NaN  \n",
      "#2        NaN       NaN  \n",
      "#21619    NaN       NaN  \n",
      "#2951     NaN       NaN  \n",
      "#2for     NaN       NaN  \n",
      "#3        NaN       NaN  \n",
      "#30       NaN       NaN  \n",
      "\n",
      "Bottom 25 words:\n",
      "                      Phrase  Characters  Terms  isStopword           Stem  \\\n",
      "zombies              zombies           7      1       False          zombi   \n",
      "zomg                    zomg           4      1       False           zomg   \n",
      "zone                    zone           4      1       False           zone   \n",
      "zones                  zones           5      1       False           zone   \n",
      "zoning                zoning           6      1       False           zone   \n",
      "zoo                      zoo           3      1       False            zoo   \n",
      "zoomed                zoomed           6      1       False           zoom   \n",
      "zooming              zooming           7      1       False           zoom   \n",
      "zotz                    zotz           4      1       False           zotz   \n",
      "zoute                  zoute           5      1       False           zout   \n",
      "zowie                  zowie           5      1       False           zowi   \n",
      "zuban                  zuban           5      1       False          zuban   \n",
      "zucchini            zucchini           8      1       False       zucchini   \n",
      "zucchinifresh  zucchinifresh          13      1       False  zucchinifresh   \n",
      "zuccini              zuccini           7      1       False        zuccini   \n",
      "zuccnini2          zuccnini2           9      1       False      zuccnini2   \n",
      "zuchini              zuchini           7      1       False        zuchini   \n",
      "zuchinni            zuchinni           8      1       False       zuchinni   \n",
      "zucini                zucini           6      1       False         zucini   \n",
      "zuco                    zuco           4      1       False           zuco   \n",
      "zulu                    zulu           4      1       False           zulu   \n",
      "zutphen              zutphen           7      1       False        zutphen   \n",
      "zyliss                zyliss           6      1       False         zyliss   \n",
      "zymbac                zymbac           6      1       False         zymbac   \n",
      "zzzs                    zzzs           4      1       False            zzz   \n",
      "\n",
      "               Count  Count Pct  Docs  Docs Pct  \n",
      "zombies          NaN        NaN   NaN       NaN  \n",
      "zomg             NaN        NaN   NaN       NaN  \n",
      "zone             NaN        NaN   NaN       NaN  \n",
      "zones            NaN        NaN   NaN       NaN  \n",
      "zoning           NaN        NaN   NaN       NaN  \n",
      "zoo              NaN        NaN   NaN       NaN  \n",
      "zoomed           NaN        NaN   NaN       NaN  \n",
      "zooming          NaN        NaN   NaN       NaN  \n",
      "zotz             NaN        NaN   NaN       NaN  \n",
      "zoute            NaN        NaN   NaN       NaN  \n",
      "zowie            NaN        NaN   NaN       NaN  \n",
      "zuban            NaN        NaN   NaN       NaN  \n",
      "zucchini         NaN        NaN   NaN       NaN  \n",
      "zucchinifresh    NaN        NaN   NaN       NaN  \n",
      "zuccini          NaN        NaN   NaN       NaN  \n",
      "zuccnini2        NaN        NaN   NaN       NaN  \n",
      "zuchini          NaN        NaN   NaN       NaN  \n",
      "zuchinni         NaN        NaN   NaN       NaN  \n",
      "zucini           NaN        NaN   NaN       NaN  \n",
      "zuco             NaN        NaN   NaN       NaN  \n",
      "zulu             NaN        NaN   NaN       NaN  \n",
      "zutphen          NaN        NaN   NaN       NaN  \n",
      "zyliss           NaN        NaN   NaN       NaN  \n",
      "zymbac           NaN        NaN   NaN       NaN  \n",
      "zzzs             NaN        NaN   NaN       NaN  \n",
      "\n",
      "Trigrams\n",
      "Number of documents: 49994\n",
      "Number of word forms (terms): 38373\n",
      "Number of words (tokens): 3084771\n",
      "Mean words per document: 61.7\n",
      "Mean term occurance: 80.4\n",
      "Number of terms occuring once: 0\n",
      "Number of terms occuring <= 5 times: 0\n",
      "Number of terms occuring <= 10 times: 3244\n",
      "Top 25 words:\n",
      "              Phrase  Characters  Terms  isStopword     Stem  Count  \\\n",
      "t                  t           1      1        True        t  32767   \n",
      "s                  s           1      1        True        s  32590   \n",
      "like            like           4      1       False     like  31624   \n",
      "good            good           4      1       False     good  24362   \n",
      "taste          taste           5      1       False     tast  23928   \n",
      "flavor        flavor           6      1       False   flavor  20996   \n",
      "just            just           4      1        True     just  18718   \n",
      "coffee        coffee           6      1       False    coffe  17959   \n",
      "great          great           5      1       False    great  15199   \n",
      "tea              tea           3      1       False      tea  14738   \n",
      "product      product           7      1       False  product  12489   \n",
      "really        really           6      1       False   realli  11950   \n",
      "love            love           4      1       False     love  10887   \n",
      "use              use           3      1       False      use  10373   \n",
      "don              don           3      1        True      don   9994   \n",
      "don t          don t           5      2       False    don t   9921   \n",
      "little        little           6      1       False    littl   9918   \n",
      "chocolate  chocolate           9      1       False   chocol   9608   \n",
      "sugar          sugar           5      1       False    sugar   8988   \n",
      "make            make           4      1       False     make   8660   \n",
      "water          water           5      1       False    water   8642   \n",
      "m                  m           1      1        True        m   8449   \n",
      "ve                ve           2      1        True       ve   8098   \n",
      "cup              cup           3      1       False      cup   7960   \n",
      "sweet          sweet           5      1       False    sweet   7879   \n",
      "\n",
      "           Count Pct   Docs  Docs Pct  \n",
      "t           0.010622  19081  0.381666  \n",
      "s           0.010565  17455  0.349142  \n",
      "like        0.010252  19669  0.393427  \n",
      "good        0.007898  17558  0.351202  \n",
      "taste       0.007757  16366  0.327359  \n",
      "flavor      0.006806  14170  0.283434  \n",
      "just        0.006068  13825  0.276533  \n",
      "coffee      0.005822   6928  0.138577  \n",
      "great       0.004927  12132  0.242669  \n",
      "tea         0.004778   4690  0.093811  \n",
      "product     0.004049   8834  0.176701  \n",
      "really      0.003874   9279  0.185602  \n",
      "love        0.003529   8861  0.177241  \n",
      "use         0.003363   7775  0.155519  \n",
      "don         0.003240   8100  0.162019  \n",
      "don t       0.003216   8045  0.160919  \n",
      "little      0.003215   7853  0.157079  \n",
      "chocolate   0.003115   4391  0.087831  \n",
      "sugar       0.002914   5511  0.110233  \n",
      "make        0.002807   7092  0.141857  \n",
      "water       0.002802   4844  0.096892  \n",
      "m           0.002739   6399  0.127995  \n",
      "ve          0.002625   6313  0.126275  \n",
      "cup         0.002580   4787  0.095751  \n",
      "sweet       0.002554   5885  0.117714  \n",
      "\n",
      "Bottom 25 words:\n",
      "                                Phrase  Characters  Terms  isStopword  \\\n",
      "new level                    new level           9      2       False   \n",
      "non instant                non instant          11      2       False   \n",
      "noodles come              noodles come          12      2       False   \n",
      "odd thing                    odd thing           9      2       False   \n",
      "noodles easy              noodles easy          12      2       False   \n",
      "occasionally drink  occasionally drink          18      2       False   \n",
      "observed                      observed           8      1       False   \n",
      "oberto                          oberto           6      1       False   \n",
      "oats chocolate          oats chocolate          14      2       False   \n",
      "oatmeal tastes          oatmeal tastes          14      2       False   \n",
      "oatmeal does              oatmeal does          12      2       False   \n",
      "nuts little                nuts little          11      2       False   \n",
      "nuts dairy wheat      nuts dairy wheat          16      3       False   \n",
      "nuts bar                      nuts bar           8      2       False   \n",
      "nutritive                    nutritive           9      1       False   \n",
      "number brands            number brands          13      2       False   \n",
      "nuking                          nuking           6      1       False   \n",
      "nt                                  nt           2      1       False   \n",
      "notices                        notices           7      1       False   \n",
      "notice s                      notice s           8      2       False   \n",
      "note like                    note like           9      2       False   \n",
      "northeast                    northeast           9      1       False   \n",
      "normal water              normal water          12      2       False   \n",
      "normal pasta              normal pasta          12      2       False   \n",
      "like hot tea              like hot tea          12      3       False   \n",
      "\n",
      "                                  Stem  Count  Count Pct  Docs  Docs Pct  \n",
      "new level                    new level     10   0.000003    10    0.0002  \n",
      "non instant                   non inst     10   0.000003    10    0.0002  \n",
      "noodles come               noodles com     10   0.000003    10    0.0002  \n",
      "odd thing                       odd th     10   0.000003    10    0.0002  \n",
      "noodles easy              noodles easi     10   0.000003    10    0.0002  \n",
      "occasionally drink  occasionally drink     10   0.000003    10    0.0002  \n",
      "observed                        observ     10   0.000003    10    0.0002  \n",
      "oberto                          oberto     10   0.000003    10    0.0002  \n",
      "oats chocolate             oats chocol     10   0.000003    10    0.0002  \n",
      "oatmeal tastes            oatmeal tast     10   0.000003    10    0.0002  \n",
      "oatmeal does                oatmeal do     10   0.000003    10    0.0002  \n",
      "nuts little                 nuts littl     10   0.000003    10    0.0002  \n",
      "nuts dairy wheat      nuts dairy wheat     10   0.000003    10    0.0002  \n",
      "nuts bar                      nuts bar     10   0.000003    10    0.0002  \n",
      "nutritive                       nutrit     10   0.000003    10    0.0002  \n",
      "number brands             number brand     10   0.000003    10    0.0002  \n",
      "nuking                            nuke     10   0.000003    10    0.0002  \n",
      "nt                                  nt     10   0.000003    10    0.0002  \n",
      "notices                          notic     10   0.000003    10    0.0002  \n",
      "notice s                       notice      10   0.000003    10    0.0002  \n",
      "note like                     note lik     10   0.000003    10    0.0002  \n",
      "northeast                    northeast     10   0.000003    10    0.0002  \n",
      "normal water                normal wat     10   0.000003    10    0.0002  \n",
      "normal pasta              normal pasta     10   0.000003    10    0.0002  \n",
      "like hot tea              like hot tea     10   0.000003    10    0.0002  \n"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"reviews_Grocery_and_Gourmet_Food_5_50000\", id_col=\"reviewID\", text_col=\"reviewText\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\st50\\AppData\\Local\\Continuum\\anaconda3\\envs\\small_sklearn\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3263: DtypeWarning: Columns (35,36) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  if (await self.run_code(code, result,  async_=asy)):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Unigrams\n"
     ]
    },
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 27.7 GiB for an array with shape (3720774528,) and data type int64",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-32f4fb10f7a1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdo_it\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_base\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"vaers2\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mid_col\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"VAERS_ID\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtext_col\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"SYMPTOM_TEXT\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_cols\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'DIED'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-3-02b31dd3b0ba>\u001b[0m in \u001b[0;36mdo_it\u001b[1;34m(file_base, id_col, text_col, target_cols)\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"\\nUnigrams\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m     \u001b[0mtdm\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvocab\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcounts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfind_counts\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtext_col\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m     \u001b[0mtdm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'out/'\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mfile_base\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'_unigram_tdm.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[0mcounts\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'out/'\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mfile_base\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'_unigram_counts.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-2-def95686fac6>\u001b[0m in \u001b[0;36mfind_counts\u001b[1;34m(data, text_col, min_df, ngrams)\u001b[0m\n\u001b[0;32m     37\u001b[0m                                 \u001b[1;34m'Count'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtdm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m                                 \u001b[1;34m'Count Pct'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtdm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mn_tokens\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 39\u001b[1;33m                                 \u001b[1;34m'Docs'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtdm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbool\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     40\u001b[0m                                 \u001b[1;34m'Docs Pct'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtdm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbool\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mn_docs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     41\u001b[0m                           })\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\small_sklearn\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mastype\u001b[1;34m(self, dtype, copy, errors)\u001b[0m\n\u001b[0;32m   5696\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5697\u001b[0m             \u001b[1;31m# else, only a single dtype is given\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 5698\u001b[1;33m             \u001b[0mnew_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   5699\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_constructor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__finalize__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5700\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\small_sklearn\\lib\\site-packages\\pandas\\core\\internals\\managers.py\u001b[0m in \u001b[0;36mastype\u001b[1;34m(self, dtype, copy, errors)\u001b[0m\n\u001b[0;32m    580\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    581\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbool\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"raise\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 582\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"astype\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    583\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    584\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mconvert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\small_sklearn\\lib\\site-packages\\pandas\\core\\internals\\managers.py\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, f, filter, **kwargs)\u001b[0m\n\u001b[0;32m    440\u001b[0m                 \u001b[0mapplied\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    441\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 442\u001b[1;33m                 \u001b[0mapplied\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    443\u001b[0m             \u001b[0mresult_blocks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_extend_blocks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mapplied\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult_blocks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    444\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\small_sklearn\\lib\\site-packages\\pandas\\core\\internals\\blocks.py\u001b[0m in \u001b[0;36mastype\u001b[1;34m(self, dtype, copy, errors)\u001b[0m\n\u001b[0;32m    621\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    622\u001b[0m             \u001b[1;31m# _astype_nansafe works fine with 1-d only\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 623\u001b[1;33m             \u001b[0mvals1d\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    624\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    625\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mastype_nansafe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvals1d\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 27.7 GiB for an array with shape (3720774528,) and data type int64"
     ]
    }
   ],
   "source": [
    "do_it(file_base=\"vaers2\", id_col=\"VAERS_ID\", text_col=\"SYMPTOM_TEXT\", target_cols=['DIED'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "do_it(file_base=\"Hillary_Emails\", id_col=\"Id\", text_col=\"ExtractedBodyText\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "small_sklearn_kernel",
   "language": "python",
   "name": "small_sklearn_kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}