textClassification_vocabulary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from textblob import TextBlob\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import pos_tag\n",
    "from gensim.models.phrases import Phrases, Phraser\n",
    "import spacy\n",
    "import textacy\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "import re\n",
    "import time\n",
    "from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(pd.read_csv('label_text.tsv', sep='\\t'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>G05B</td>\n",
       "      <td>an apparatus for generating a saddle shaped tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>H01L</td>\n",
       "      <td>an apparatus for generating a saddle shaped tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A61M</td>\n",
       "      <td>the present invention provides apparatus and m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A01K</td>\n",
       "      <td>a restraint system for an animal comprising a ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B29C</td>\n",
       "      <td>a container or tray having various features th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                               text\n",
       "0   G05B  an apparatus for generating a saddle shaped tr...\n",
       "1   H01L  an apparatus for generating a saddle shaped tr...\n",
       "2   A61M  the present invention provides apparatus and m...\n",
       "3   A01K  a restraint system for an animal comprising a ...\n",
       "4   B29C  a container or tray having various features th..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check the distribution of classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "label_cnts = df.groupby(['label']).count().reset_index().rename(columns={'text':'count'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1256.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2133.288217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>9346.182892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>92.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>333.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1370.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>262272.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               count\n",
       "count    1256.000000\n",
       "mean     2133.288217\n",
       "std      9346.182892\n",
       "min         1.000000\n",
       "25%        92.750000\n",
       "50%       333.000000\n",
       "75%      1370.000000\n",
       "max    262272.000000"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_cnts.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Choose n samples for each class, since the distribution is imbalanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "size = 100        # sample size\n",
    "replace = True  # cannot choose False, since some classes do not have 100 samples\n",
    "fn = lambda obj: obj.loc[np.random.choice(obj.index, size, replace),:]\n",
    "sub_df = df.groupby('label', as_index=False).apply(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "125600"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sub_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Customize a vocabulary from patent data, including words and phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopwords2 = [\"a\",\"about\",\"above\",\"after\",\"again\",\"against\",\"ain\",\"all\",\"am\",\"an\",\"and\",\"any\",\"are\",\"aren\",\"aren't\",\"as\",\"at\",\"be\",\"because\",\"been\",\"before\",\"being\",\"below\",\"between\",\"both\",\"but\",\"by\",\"can\",\"couldn\",\"couldn't\",\"d\",\"did\",\"didn\",\"didn't\",\"do\",\"does\",\"doesn\",\"doesn't\",\"doing\",\"don\",\"don't\",\"down\",\"during\",\"each\",\"few\",\"for\",\"from\",\"further\",\"had\",\"hadn\",\"hadn't\",\"has\",\"hasn\",\"hasn't\",\"have\",\"haven\",\"haven't\",\"having\",\"he\",\"her\",\"here\",\"hers\",\"herself\",\"him\",\"himself\",\"his\",\"how\",\"i\",\"if\",\"in\",\"into\",\"is\",\"isn\",\"isn't\",\"it\",\"it's\",\"its\",\"itself\",\"just\",\"ll\",\"m\",\"ma\",\"me\",\"mightn\",\"mightn't\",\"more\",\"most\",\"mustn\",\"mustn't\",\"my\",\"myself\",\"needn\",\"needn't\",\"no\",\"nor\",\"not\",\"now\",\"o\",\"of\",\"off\",\"on\",\"once\",\"only\",\"or\",\"other\",\"our\",\"ours\",\"ourselves\",\"out\",\"over\",\"own\",\"re\",\"s\",\"same\",\"shan\",\"shan't\",\"she\",\"she's\",\"should\",\"should've\",\"shouldn\",\"shouldn't\",\"so\",\"some\",\"such\",\"t\",\"than\",\"that\",\"that'll\",\"the\",\"their\",\"theirs\",\"them\",\"themselves\",\"then\",\"there\",\"these\",\"they\",\"this\",\"those\",\"through\",\"to\",\"too\",\"under\",\"until\",\"up\",\"ve\",\"very\",\"was\",\"wasn\",\"wasn't\",\"we\",\"were\",\"weren\",\"weren't\",\"what\",\"when\",\"where\",\"which\",\"while\",\"who\",\"whom\",\"why\",\"will\",\"with\",\"won\",\"won't\",\"wouldn\",\"wouldn't\",\"y\",\"you\",\"you'd\",\"you'll\",\"you're\",\"you've\",\"your\",\"yours\",\"yourself\",\"yourselves\",\"could\",\"he'd\",\"he'll\",\"he's\",\"here's\",\"how's\",\"i'd\",\"i'll\",\"i'm\",\"i've\",\"let's\",\"ought\",\"she'd\",\"she'll\",\"that's\",\"there's\",\"they'd\",\"they'll\",\"they're\",\"they've\",\"we'd\",\"we'll\",\"we're\",\"we've\",\"what's\",\"when's\",\"where's\",\"who's\",\"why's\",\"would\",\"able\",\"abst\",\"accordance\",\"according\",\"accordingly\",\"across\",\"act\",\"actually\",\"added\",\"adj\",\"affected\",\"affecting\",\"affects\",\"afterwards\",\"ah\",\"almost\",\"alone\",\"along\",\"already\",\"also\",\"although\",\"always\",\"among\",\"amongst\",\"announce\",\"another\",\"anybody\",\"anyhow\",\"anymore\",\"anyone\",\"anything\",\"anyway\",\"anyways\",\"anywhere\",\"apparently\",\"approximately\",\"arent\",\"arise\",\"around\",\"aside\",\"ask\",\"asking\",\"auth\",\"available\",\"away\",\"awfully\",\"b\",\"back\",\"became\",\"become\",\"becomes\",\"becoming\",\"beforehand\",\"begin\",\"beginning\",\"beginnings\",\"begins\",\"behind\",\"believe\",\"beside\",\"besides\",\"beyond\",\"biol\",\"brief\",\"briefly\",\"c\",\"ca\",\"came\",\"cannot\",\"can't\",\"cause\",\"causes\",\"certain\",\"certainly\",\"co\",\"com\",\"come\",\"comes\",\"contain\",\"containing\",\"contains\",\"couldnt\",\"date\",\"different\",\"done\",\"downwards\",\"due\",\"e\",\"ed\",\"edu\",\"effect\",\"eg\",\"eight\",\"eighty\",\"either\",\"else\",\"elsewhere\",\"end\",\"ending\",\"enough\",\"especially\",\"et\",\"etc\",\"even\",\"ever\",\"every\",\"everybody\",\"everyone\",\"everything\",\"everywhere\",\"ex\",\"except\",\"f\",\"far\",\"ff\",\"fifth\",\"first\",\"five\",\"fix\",\"followed\",\"following\",\"follows\",\"former\",\"formerly\",\"forth\",\"found\",\"four\",\"furthermore\",\"g\",\"gave\",\"get\",\"gets\",\"getting\",\"give\",\"given\",\"gives\",\"giving\",\"go\",\"goes\",\"gone\",\"got\",\"gotten\",\"h\",\"happens\",\"hardly\",\"hed\",\"hence\",\"hereafter\",\"hereby\",\"herein\",\"heres\",\"hereupon\",\"hes\",\"hi\",\"hid\",\"hither\",\"home\",\"howbeit\",\"however\",\"hundred\",\"id\",\"ie\",\"im\",\"immediate\",\"immediately\",\"importance\",\"important\",\"inc\",\"indeed\",\"index\",\"information\",\"instead\",\"invention\",\"inward\",\"itd\",\"it'll\",\"j\",\"k\",\"keep\",\"keeps\",\"kept\",\"kg\",\"km\",\"know\",\"known\",\"knows\",\"l\",\"largely\",\"last\",\"lately\",\"later\",\"latter\",\"latterly\",\"least\",\"less\",\"lest\",\"let\",\"lets\",\"like\",\"liked\",\"likely\",\"line\",\"little\",\"'ll\",\"look\",\"looking\",\"looks\",\"ltd\",\"made\",\"mainly\",\"make\",\"makes\",\"many\",\"may\",\"maybe\",\"mean\",\"means\",\"meantime\",\"meanwhile\",\"merely\",\"mg\",\"might\",\"million\",\"miss\",\"ml\",\"moreover\",\"mostly\",\"mr\",\"mrs\",\"much\",\"mug\",\"must\",\"n\",\"na\",\"name\",\"namely\",\"nay\",\"nd\",\"near\",\"nearly\",\"necessarily\",\"necessary\",\"need\",\"needs\",\"neither\",\"never\",\"nevertheless\",\"new\",\"next\",\"nine\",\"ninety\",\"nobody\",\"non\",\"none\",\"nonetheless\",\"noone\",\"normally\",\"nos\",\"noted\",\"nothing\",\"nowhere\",\"obtain\",\"obtained\",\"obviously\",\"often\",\"oh\",\"ok\",\"okay\",\"old\",\"omitted\",\"one\",\"ones\",\"onto\",\"ord\",\"others\",\"otherwise\",\"outside\",\"overall\",\"owing\",\"p\",\"page\",\"pages\",\"part\",\"particular\",\"particularly\",\"past\",\"per\",\"perhaps\",\"placed\",\"please\",\"plus\",\"poorly\",\"possible\",\"possibly\",\"potentially\",\"pp\",\"predominantly\",\"present\",\"previously\",\"primarily\",\"probably\",\"promptly\",\"proud\",\"provides\",\"put\",\"q\",\"que\",\"quickly\",\"quite\",\"qv\",\"r\",\"ran\",\"rather\",\"rd\",\"readily\",\"really\",\"recent\",\"recently\",\"ref\",\"refs\",\"regarding\",\"regardless\",\"regards\",\"related\",\"relatively\",\"research\",\"respectively\",\"resulted\",\"resulting\",\"results\",\"right\",\"run\",\"said\",\"saw\",\"say\",\"saying\",\"says\",\"sec\",\"section\",\"see\",\"seeing\",\"seem\",\"seemed\",\"seeming\",\"seems\",\"seen\",\"self\",\"selves\",\"sent\",\"seven\",\"several\",\"shall\",\"shed\",\"shes\",\"show\",\"showed\",\"shown\",\"showns\",\"shows\",\"significant\",\"significantly\",\"similar\",\"similarly\",\"since\",\"six\",\"slightly\",\"somebody\",\"somehow\",\"someone\",\"somethan\",\"something\",\"sometime\",\"sometimes\",\"somewhat\",\"somewhere\",\"soon\",\"sorry\",\"specifically\",\"specified\",\"specify\",\"specifying\",\"still\",\"stop\",\"strongly\",\"sub\",\"substantially\",\"successfully\",\"sufficiently\",\"suggest\",\"sup\",\"sure\",\"take\",\"taken\",\"taking\",\"tell\",\"tends\",\"th\",\"thank\",\"thanks\",\"thanx\",\"thats\",\"that've\",\"thence\",\"thereafter\",\"thereby\",\"thered\",\"therefore\",\"therein\",\"there'll\",\"thereof\",\"therere\",\"theres\",\"thereto\",\"thereupon\",\"there've\",\"theyd\",\"theyre\",\"think\",\"thou\",\"though\",\"thoughh\",\"thousand\",\"throug\",\"throughout\",\"thru\",\"thus\",\"til\",\"tip\",\"together\",\"took\",\"toward\",\"towards\",\"tried\",\"tries\",\"truly\",\"try\",\"trying\",\"ts\",\"twice\",\"two\",\"u\",\"un\",\"unfortunately\",\"unless\",\"unlike\",\"unlikely\",\"unto\",\"upon\",\"ups\",\"us\",\"use\",\"used\",\"useful\",\"usefully\",\"usefulness\",\"uses\",\"using\",\"usually\",\"v\",\"value\",\"various\",\"'ve\",\"via\",\"viz\",\"vol\",\"vols\",\"vs\",\"w\",\"want\",\"wants\",\"wasnt\",\"way\",\"wed\",\"welcome\",\"went\",\"werent\",\"whatever\",\"what'll\",\"whats\",\"whence\",\"whenever\",\"whereafter\",\"whereas\",\"whereby\",\"wherein\",\"wheres\",\"whereupon\",\"wherever\",\"whether\",\"whim\",\"whither\",\"whod\",\"whoever\",\"whole\",\"who'll\",\"whomever\",\"whos\",\"whose\",\"widely\",\"willing\",\"wish\",\"within\",\"without\",\"wont\",\"words\",\"world\",\"wouldnt\",\"www\",\"x\",\"yes\",\"yet\",\"youd\",\"youre\",\"z\",\"zero\",\"a's\",\"ain't\",\"allow\",\"allows\",\"apart\",\"appear\",\"appreciate\",\"appropriate\",\"associated\",\"best\",\"better\",\"c'mon\",\"c's\",\"cant\",\"changes\",\"clearly\",\"concerning\",\"consequently\",\"consider\",\"considering\",\"corresponding\",\"course\",\"currently\",\"definitely\",\"described\",\"despite\",\"entirely\",\"exactly\",\"example\",\"going\",\"greetings\",\"hello\",\"help\",\"hopefully\",\"ignored\",\"inasmuch\",\"indicate\",\"indicated\",\"indicates\",\"inner\",\"insofar\",\"it'd\",\"keep\",\"keeps\",\"novel\",\"presumably\",\"reasonably\",\"second\",\"secondly\",\"sensible\",\"serious\",\"seriously\",\"sure\",\"t's\",\"third\",\"thorough\",\"thoroughly\",\"three\",\"well\",\"wonder\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopwords = stopwords.words(\"english\") + list(ENGLISH_STOP_WORDS) + stopwords2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getWords_lemmatization(words):\n",
    "    lem = WordNetLemmatizer()\n",
    "    words = [lem.lemmatize(word) for word in words]\n",
    "    return words\n",
    "\n",
    "def getNoun_phrase_Textblob(text):\n",
    "    blob = TextBlob(text)\n",
    "    textblob_words = set(getWords_lemmatization(list(blob.noun_phrases)))\n",
    "    return textblob_words\n",
    "# for item in text:\n",
    "#     print (getNoun_phrase_Textblob(item))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getNounPhrases_textRank(doc_f, nouns):\n",
    "    text_rank_result = textrank(doc_f, normalize='lemma')\n",
    "    for t in text_rank_result:\n",
    "        p = t[0]\n",
    "        phrase_no_stopwords = ' '.join([word for word in p.split(' ') if word not in stopwords])\n",
    "        nouns.add(phrase_no_stopwords)\n",
    "    return nouns  \n",
    "def getVerb(sent, verbs):\n",
    "# #     pattern = r'(<VERB>?<ADV>*<VERB>+)'\n",
    "# #     verb_phrases = textacy.extract.pos_regex_matches(doc_f, pattern)\n",
    "# #     for chunk in verb_phrases:\n",
    "# #         print (chunk.text)\n",
    "# #         verbs.add(chunk.text)\n",
    "#     #print (verbs)\n",
    "\n",
    "    #use nltk\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    pos_tagged = pos_tag(word_tokenize(sent))\n",
    "    verb_list = [lemmatizer.lemmatize(word, 'v') for word, pos in pos_tagged if (pos == 'VBP') or (pos == 'VBN') or (pos == 'VBD')]\n",
    "    verb_list = [word for word in verb_list if word not in stopwords]\n",
    "    verbs = verbs.union(set(verb_list))\n",
    "    return verbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text = sub_df['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 2864.3837723731995 seconds ---\n"
     ]
    }
   ],
   "source": [
    "nouns = set()\n",
    "verbs = set()\n",
    "start_time = time.time()\n",
    "\n",
    "for sentence in text:\n",
    "    doc_f = textacy.make_spacy_doc(sentence, lang='en_core_web_sm')\n",
    "    \n",
    "    #spacy's default noun phrases\n",
    "    #text_doc = nlp(item)\n",
    "    #for chunk in text_doc.noun_chunks:\n",
    "    #    chunk_no_stopwords = ' '.join([word for word in chunk.text.split(' ') if word not in stopwords])\n",
    "    #    temp.add(chunk_no_stopwords)\n",
    "    \n",
    "    #textacy's keyterm extraction algorithm - sgrank\n",
    "    #print (sgrank(textacy.make_spacy_doc(item, lang='en_core_web_sm')))\n",
    "    \n",
    "    #textacy's keyterm extraction algorithm - textrank\n",
    "    nouns = getNounPhrases_textRank(doc_f, nouns)\n",
    "    #verbs = getVerb(sentence, verbs)\n",
    "    \n",
    "#print (verbs)\n",
    "#print (nouns)\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "customize_vocabulary = {}\n",
    "idx = 0\n",
    "for word in nouns:\n",
    "    customize_vocabulary[word] = idx\n",
    "    idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "350106"
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(customize_vocabulary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate tfidf vectorization according to the own vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0   313189     a drive torque modulation is generated in resp...\n",
       "    6777       the invention relates to a method for detectin...\n",
       "    2089499    a vehicle has a rear wheel drive device for dr...\n",
       "    638418     plant eradication using by inflicting upon a p...\n",
       "    99240      a vehicle system for providing an interface fo...\n",
       "                                     ...                        \n",
       "99  1298209    the invention refers to a screen star in parti...\n",
       "    1580093    this device is a vehicle for scraping earth fr...\n",
       "    69868      a compact powder case includes a top case a bo...\n",
       "    316100     embodiments herein describe a composition incl...\n",
       "    1463544    the invention relates to a process for size cl...\n",
       "Name: text, Length: 10000, dtype: object"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = sub_df[0:10000]['text']\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 44.04098200798035 seconds ---\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer(vocabulary=customize_vocabulary, ngram_range=(1,5))\n",
    "tfidf_vectors = tfidf_vectorizer.fit_transform(sub_df['text'])\n",
    "\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(125600, 350105)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_vectors.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py3-tf2]",
   "language": "python",
   "name": "conda-env-py3-tf2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}