textClassification_sequenceNNs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use pre-trained word embeddings based on patent data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word2vec_model = KeyedVectors.load_word2vec_format('uspto_2m_abstract_word2vec.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#read text data and their labels\n",
    "label_words_df = pd.DataFrame(pd.read_csv('uspto_2m_abstr_label_valid_label.tsv', sep='\\t'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Abstract</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>an apparatus for generating a saddle shaped tr...</td>\n",
       "      <td>H01L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a container or tray having various features th...</td>\n",
       "      <td>B65D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>screening methods for identifying compounds an...</td>\n",
       "      <td>A61K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>methods of forming conductive pattern structur...</td>\n",
       "      <td>H01L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a method of logging information about events f...</td>\n",
       "      <td>G07C</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Abstract  label\n",
       "0  an apparatus for generating a saddle shaped tr...   H01L\n",
       "1  a container or tray having various features th...   B65D\n",
       "2  screening methods for identifying compounds an...   A61K\n",
       "3  methods of forming conductive pattern structur...   H01L\n",
       "4  a method of logging information about events f...   G07C"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_words_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "315000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#set up balanced dataset\n",
    "size = 500        # sample size\n",
    "replace = True  # cannot choose False, since some classes do not have 100 samples\n",
    "fn = lambda obj: obj.loc[np.random.choice(obj.index, size, replace),:]\n",
    "sub_df = label_words_df.groupby('label', as_index=False).apply(fn)\n",
    "y = sub_df['label']\n",
    "len(sub_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Abstract', 'label'], dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique tokens:  63929\n"
     ]
    }
   ],
   "source": [
    "#extract words\n",
    "from tensorflow.python.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.python.keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "tokenizer = Tokenizer()\n",
    "#tokenizer = Tokenizer(num_words=20)\n",
    "tokenizer.fit_on_texts(sub_df['Abstract'].values)\n",
    "sequences = tokenizer.texts_to_sequences(sub_df['Abstract'].values)\n",
    "\n",
    "\n",
    "#max length for pad sequences\n",
    "seq_max_len = max([len(s.split()) for s in sub_df['Abstract'].values])\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print ('Number of unique tokens: ', len(word_index))\n",
    "\n",
    "x_pad = pad_sequences(sequences, maxlen=seq_max_len)\n",
    "encoder = LabelEncoder()\n",
    "y_set = encoder.fit_transform(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare the embedding layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding_dim = word2vec_model.vector_size\n",
    "embedding_matrix = np.zeros((len(word_index) + 1, embedding_dim))\n",
    "for word, i in word_index.items():\n",
    "    if word in word2vec_model:\n",
    "        embedding_matrix[i] = word2vec_model[word]\n",
    "    #print (word, i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train LSTM, GRU, CNN with the pre-trained embedding layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CNN is faster than LSTM, GRU.\n",
    "\n",
    "Future work:\n",
    "\n",
    "1. Try seqCNN, etc., other CNN variations.\n",
    "2. Try large dataset using CNN, compared to Naive Bayes, SVM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.python.keras.layers import Embedding, Dense, LSTM, GRU\n",
    "from tensorflow.python.keras.layers import Conv1D, MaxPooling1D, Flatten\n",
    "from tensorflow.python.keras.layers import Dropout, SeparableConv1D, GlobalAveragePooling1D\n",
    "from tensorflow.python.keras import models, callbacks\n",
    "from tensorflow.python.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numOfclasses = len(set(y_set))\n",
    "last_layer_activation = 'softmax'\n",
    "hidden_layer_activation = 'relu'\n",
    "dropout_rate = 0.2\n",
    "loss = 'sparse_categorical_crossentropy'\n",
    "learning_rate = 0.001\n",
    "epochs = 2\n",
    "n_layers = 1\n",
    "units = 64\n",
    "batch_size = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 283500 samples, validate on 31500 samples\n",
      "Epoch 1/2\n",
      "283500/283500 [==============================] - 1687s 6ms/sample - loss: 5.7565 - accuracy: 0.0183 - val_loss: 13.9873 - val_accuracy: 0.0000e+00\n",
      "Epoch 2/2\n",
      "283500/283500 [==============================] - 1738s 6ms/sample - loss: 5.0057 - accuracy: 0.0579 - val_loss: 15.2838 - val_accuracy: 0.0000e+00\n"
     ]
    }
   ],
   "source": [
    "#LSTM\n",
    "#for output layer\n",
    "model = models.Sequential()\n",
    "#use pre-trained embedding matrix\n",
    "model.add(Embedding(len(word_index) + 1, embedding_dim, weights=[embedding_matrix], \n",
    "                    input_length=seq_max_len, trainable=False))\n",
    "\n",
    "model.add(LSTM(units=units, dropout=dropout_rate, recurrent_dropout=dropout_rate))\n",
    "model.add(Dense(units=numOfclasses, activation=last_layer_activation))\n",
    "model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(x_pad, y_set, epochs=epochs, batch_size=batch_size, validation_split=0.1, \n",
    "                    verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 283500 samples, validate on 31500 samples\n",
      "283500/283500 [==============================] - 1608s 6ms/sample - loss: 5.4182 - accuracy: 0.0425 - val_loss: 14.3833 - val_accuracy: 0.0000e+00\n"
     ]
    }
   ],
   "source": [
    "#GRU\n",
    "epochs = 1\n",
    "model = models.Sequential()\n",
    "#use pre-trained embedding matrix\n",
    "model.add(Embedding(len(word_index) + 1, embedding_dim, weights=[embedding_matrix], \n",
    "                    input_length=seq_max_len, trainable=False))\n",
    "\n",
    "model.add(GRU(units=units, dropout=dropout_rate, recurrent_dropout=dropout_rate))\n",
    "model.add(Dense(units=numOfclasses, activation=last_layer_activation))\n",
    "model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(x_pad, y_set, epochs=epochs, batch_size=batch_size, validation_split=0.1, \n",
    "                    verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 283500 samples, validate on 31500 samples\n",
      "Epoch 1/2\n",
      "283500/283500 [==============================] - 842s 3ms/sample - loss: 2.8366 - accuracy: 0.4606 - val_loss: 15.6995 - val_accuracy: 0.0000e+00\n",
      "Epoch 2/2\n",
      "283500/283500 [==============================] - 840s 3ms/sample - loss: 1.2347 - accuracy: 0.7308 - val_loss: 18.4088 - val_accuracy: 0.0000e+00\n"
     ]
    }
   ],
   "source": [
    "#CNN\n",
    "model = models.Sequential()\n",
    "model.add(Embedding(len(word_index) + 1, embedding_dim, weights=[embedding_matrix], \n",
    "                    input_length=seq_max_len, trainable=False))\n",
    "model.add(Conv1D(250, 5, activation=hidden_layer_activation))\n",
    "model.add(MaxPooling1D(pool_size=4))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(units=numOfclasses, activation=last_layer_activation))\n",
    "model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(x_pad, y_set, epochs=epochs, batch_size=batch_size, validation_split=0.1, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(x_pad, y_set, test_size = 0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 252000 samples, validate on 63000 samples\n",
      "Epoch 1/20\n",
      "252000/252000 [==============================] - 510s 2ms/sample - loss: 2.9933 - accuracy: 0.4491 - val_loss: 2.0919 - val_accuracy: 0.5856\n",
      "Epoch 2/20\n",
      "252000/252000 [==============================] - 551s 2ms/sample - loss: 1.2979 - accuracy: 0.7330 - val_loss: 1.9458 - val_accuracy: 0.6271\n",
      "Epoch 3/20\n",
      "252000/252000 [==============================] - 545s 2ms/sample - loss: 0.9508 - accuracy: 0.8056 - val_loss: 1.9214 - val_accuracy: 0.6333\n",
      "Epoch 4/20\n",
      "252000/252000 [==============================] - 549s 2ms/sample - loss: 0.8053 - accuracy: 0.8299 - val_loss: 1.9267 - val_accuracy: 0.6381\n",
      "Epoch 5/20\n",
      "252000/252000 [==============================] - 545s 2ms/sample - loss: 0.7241 - accuracy: 0.8378 - val_loss: 1.9210 - val_accuracy: 0.6405\n",
      "Epoch 6/20\n",
      "252000/252000 [==============================] - 547s 2ms/sample - loss: 0.6618 - accuracy: 0.8418 - val_loss: 1.9227 - val_accuracy: 0.6401\n",
      "Epoch 7/20\n",
      "252000/252000 [==============================] - 549s 2ms/sample - loss: 0.6179 - accuracy: 0.8446 - val_loss: 1.9467 - val_accuracy: 0.6393\n",
      "Epoch 8/20\n",
      "252000/252000 [==============================] - 545s 2ms/sample - loss: 0.5806 - accuracy: 0.8463 - val_loss: 1.9478 - val_accuracy: 0.6409\n",
      "Epoch 9/20\n",
      "252000/252000 [==============================] - 542s 2ms/sample - loss: 0.5521 - accuracy: 0.8470 - val_loss: 1.9324 - val_accuracy: 0.6380\n",
      "Epoch 10/20\n",
      "252000/252000 [==============================] - 548s 2ms/sample - loss: 0.5265 - accuracy: 0.8479 - val_loss: 1.9343 - val_accuracy: 0.6399\n",
      "Epoch 11/20\n",
      "252000/252000 [==============================] - 547s 2ms/sample - loss: 0.5036 - accuracy: 0.8496 - val_loss: 1.9924 - val_accuracy: 0.6393\n",
      "Epoch 12/20\n",
      "252000/252000 [==============================] - 544s 2ms/sample - loss: 0.4856 - accuracy: 0.8496 - val_loss: 2.0014 - val_accuracy: 0.6393\n",
      "Epoch 13/20\n",
      "252000/252000 [==============================] - 551s 2ms/sample - loss: 0.4688 - accuracy: 0.8503 - val_loss: 2.0270 - val_accuracy: 0.6371\n",
      "Epoch 14/20\n",
      "252000/252000 [==============================] - 553s 2ms/sample - loss: 0.4533 - accuracy: 0.8520 - val_loss: 2.0203 - val_accuracy: 0.6379\n",
      "Epoch 15/20\n",
      "252000/252000 [==============================] - 549s 2ms/sample - loss: 0.4406 - accuracy: 0.8531 - val_loss: 2.0477 - val_accuracy: 0.6403\n",
      "Epoch 16/20\n",
      "252000/252000 [==============================] - 551s 2ms/sample - loss: 0.4277 - accuracy: 0.8540 - val_loss: 2.0910 - val_accuracy: 0.6400\n",
      "Epoch 17/20\n",
      "252000/252000 [==============================] - 551s 2ms/sample - loss: 0.4162 - accuracy: 0.8543 - val_loss: 2.0791 - val_accuracy: 0.6408\n",
      "Epoch 18/20\n",
      "252000/252000 [==============================] - 549s 2ms/sample - loss: 0.4075 - accuracy: 0.8552 - val_loss: 2.0999 - val_accuracy: 0.6400\n",
      "Epoch 19/20\n",
      "252000/252000 [==============================] - 554s 2ms/sample - loss: 0.3980 - accuracy: 0.8559 - val_loss: 2.1337 - val_accuracy: 0.6391\n",
      "Epoch 20/20\n",
      "252000/252000 [==============================] - 557s 2ms/sample - loss: 0.3899 - accuracy: 0.8564 - val_loss: 2.1605 - val_accuracy: 0.6403\n"
     ]
    }
   ],
   "source": [
    "#adding more layers get worse results\n",
    "epochs = 20\n",
    "model = models.Sequential()\n",
    "model.add(Embedding(len(word_index) + 1, embedding_dim, weights=[embedding_matrix], \n",
    "                    input_length=seq_max_len, trainable=False))\n",
    "model.add(Conv1D(128, 3, activation=hidden_layer_activation))\n",
    "model.add(MaxPooling1D(pool_size=2))  #vary 2,3,4,5\n",
    "model.add(Flatten())\n",
    "model.add(Dense(units=numOfclasses, activation=last_layer_activation))\n",
    "model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_test, y_test), verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Train CNN and allows to learn the embedding layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 252000 samples, validate on 63000 samples\n",
      "Epoch 1/20\n",
      "252000/252000 [==============================] - 718s 3ms/sample - loss: 3.4053 - accuracy: 0.4093 - val_loss: 2.2308 - val_accuracy: 0.5702\n",
      "Epoch 2/20\n",
      "252000/252000 [==============================] - 718s 3ms/sample - loss: 1.3872 - accuracy: 0.7234 - val_loss: 2.0174 - val_accuracy: 0.6224\n",
      "Epoch 3/20\n",
      "252000/252000 [==============================] - 719s 3ms/sample - loss: 0.9614 - accuracy: 0.8102 - val_loss: 2.0157 - val_accuracy: 0.6290\n",
      "Epoch 4/20\n",
      "252000/252000 [==============================] - 728s 3ms/sample - loss: 0.8019 - accuracy: 0.8343 - val_loss: 1.9627 - val_accuracy: 0.6305\n",
      "Epoch 5/20\n",
      "252000/252000 [==============================] - 727s 3ms/sample - loss: 0.7086 - accuracy: 0.8416 - val_loss: 1.9531 - val_accuracy: 0.6352\n",
      "Epoch 6/20\n",
      "252000/252000 [==============================] - 728s 3ms/sample - loss: 0.6447 - accuracy: 0.8447 - val_loss: 1.9782 - val_accuracy: 0.6325\n",
      "Epoch 7/20\n",
      "252000/252000 [==============================] - 728s 3ms/sample - loss: 0.5942 - accuracy: 0.8467 - val_loss: 1.9596 - val_accuracy: 0.6338\n",
      "Epoch 8/20\n",
      "252000/252000 [==============================] - 729s 3ms/sample - loss: 0.5552 - accuracy: 0.8468 - val_loss: 1.9910 - val_accuracy: 0.6350\n",
      "Epoch 9/20\n",
      "252000/252000 [==============================] - 730s 3ms/sample - loss: 0.5238 - accuracy: 0.8480 - val_loss: 1.9831 - val_accuracy: 0.6345\n",
      "Epoch 10/20\n",
      "252000/252000 [==============================] - 730s 3ms/sample - loss: 0.4948 - accuracy: 0.8493 - val_loss: 2.0379 - val_accuracy: 0.6341\n",
      "Epoch 11/20\n",
      "252000/252000 [==============================] - 729s 3ms/sample - loss: 0.4705 - accuracy: 0.8508 - val_loss: 2.0554 - val_accuracy: 0.6343\n",
      "Epoch 12/20\n",
      "252000/252000 [==============================] - 733s 3ms/sample - loss: 0.4510 - accuracy: 0.8505 - val_loss: 2.0859 - val_accuracy: 0.6377\n",
      "Epoch 13/20\n",
      "252000/252000 [==============================] - 735s 3ms/sample - loss: 0.4314 - accuracy: 0.8528 - val_loss: 2.1338 - val_accuracy: 0.6357\n",
      "Epoch 14/20\n",
      "252000/252000 [==============================] - 736s 3ms/sample - loss: 0.4167 - accuracy: 0.8535 - val_loss: 2.2174 - val_accuracy: 0.6354\n",
      "Epoch 15/20\n",
      "252000/252000 [==============================] - 737s 3ms/sample - loss: 0.4035 - accuracy: 0.8537 - val_loss: 2.2552 - val_accuracy: 0.6360\n",
      "Epoch 16/20\n",
      "252000/252000 [==============================] - 735s 3ms/sample - loss: 0.3912 - accuracy: 0.8550 - val_loss: 2.2570 - val_accuracy: 0.6355\n",
      "Epoch 17/20\n",
      "252000/252000 [==============================] - 732s 3ms/sample - loss: 0.3807 - accuracy: 0.8562 - val_loss: 2.3492 - val_accuracy: 0.6350\n",
      "Epoch 18/20\n",
      "252000/252000 [==============================] - 720s 3ms/sample - loss: 0.3713 - accuracy: 0.8572 - val_loss: 2.3975 - val_accuracy: 0.6364\n",
      "Epoch 19/20\n",
      "252000/252000 [==============================] - 714s 3ms/sample - loss: 0.3631 - accuracy: 0.8577 - val_loss: 2.4405 - val_accuracy: 0.6359\n",
      "Epoch 20/20\n",
      "252000/252000 [==============================] - 739s 3ms/sample - loss: 0.3552 - accuracy: 0.8589 - val_loss: 2.4892 - val_accuracy: 0.6354\n"
     ]
    }
   ],
   "source": [
    "#CNN without weights at the embedding layer\n",
    "model = models.Sequential()\n",
    "model.add(Embedding(len(word_index) + 1, embedding_dim, input_length=seq_max_len))\n",
    "model.add(Conv1D(128, 3, activation=hidden_layer_activation))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(units=numOfclasses, activation=last_layer_activation))\n",
    "model.compile(loss=loss, optimizer='adam', metrics=['accuracy'])\n",
    "history = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_test, y_test), verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is no big difference between using a pre-trained embedding layer or not, for CNN.\n",
    "\n",
    "With the pre-trained embedding layer, LSTM or GRU does not perform well. Since it takes a long time using these two methods, we did not attempt them with learning the embedding layer. Later, we can add more layers to optimize these neural networks."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py3-tf2]",
   "language": "python",
   "name": "conda-env-py3-tf2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}