lstm_word2vec_archive.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code in this notebook is based on the [Keras documentation](https://keras.io/) and [blog](https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html) as well as this [word2vec tutorial](http://adventuresinmachinelearning.com/gensim-word2vec-tutorial/).   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using CNTK backend\n"
     ]
    }
   ],
   "source": [
    "import gzip\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import requests\n",
    "import time\n",
    "\n",
    "os.environ['KERAS_BACKEND']='cntk'\n",
    "from keras.preprocessing import sequence\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "from keras.models import Sequential, load_model\n",
    "from keras import regularizers\n",
    "from keras.optimizers import SGD\n",
    "from keras.layers import Dense, Dropout, Embedding, LSTM, Bidirectional\n",
    "from keras.callbacks import History, CSVLogger\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download the Amazon reviews data for food from the Internet archive \n",
    "[J. McAuley and J. Leskovec. Hidden factors and hidden topics: understanding rating dimensions with review text. RecSys, 2013]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get(\"https://archive.org/download/amazon-reviews-1995-2013/Gourmet_Foods.txt.gz\")\n",
    "with open(\"Gourmet_Foods.txt.gz\", 'wb') as fp:\n",
    "    fp.write(r.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open(\"Gourmet_Foods.txt.gz\", \"rb\") as fp:\n",
    "    file_content = fp.read()\n",
    "s = file_content.decode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"Gourmet_Foods.txt\", \"w\") as fp:\n",
    "    fp.write(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "lst = s.split('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract scores and review texts from file "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_lst = lst[9:len(lst):11]\n",
    "score_lst = lst[6:len(lst):11]\n",
    "score_lst2 = [sc[14:17] for sc in score_lst]\n",
    "text_lst2 = [txt[13:] for txt in text_lst]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = pd.DataFrame(data={'text': text_lst2, 'rating': score_lst2})\n",
    "all_data.loc[:, 'rating'] = all_data['rating'].astype(float)\n",
    "all_data.loc[:, 'rating'] = all_data['rating'].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove medium rating and convert to binary classification (high vs. low rating).  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = all_data[all_data['rating'] != 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "new_data = all_data.replace({'rating': {1: '0', 2: '0', 4: '1', 5: '1'}})\n",
    "new_data.loc[:, 'rating'] = new_data['rating'].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract a balanced subsample and split into training and test sets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_data = pd.concat([new_data[new_data.rating == 0].sample(10000), new_data[new_data.rating == 1].sample(10000)])\n",
    "shuffled = sample_data.iloc[np.random.permutation(20000), :]\n",
    "train_data = shuffled.iloc[:10000, :]\n",
    "test_data = shuffled.iloc[10000:, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    5020\n",
       "1    4980\n",
       "Name: rating, dtype: int64"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.rating.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    5020\n",
       "0    4980\n",
       "Name: rating, dtype: int64"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.rating.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set the dimensions of the input and the embedding. \n",
    "\n",
    "MAX_DOC_LENGTH : the size of the input i.e. the number of words in the document. Longer documents will be truncated, shorter ones will be padded with zeros.\n",
    "\n",
    "VOCAB_SIZE : the size of the word encoding (number of most frequent words to keep in the vocabulary)\n",
    "\n",
    "EMBEDDING_DIM : the dimensionality of the word embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_DOC_LEN = 300\n",
    "VOCAB_SIZE = 6000\n",
    "EMBEDDING_DIM = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TEXT_COL = 'text'\n",
    "LABEL_COL = 'rating'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit a Keras tokenizer to the most frequent words using the entire training data set as the corpus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# tokenize, create seqs, pad\n",
    "tok = Tokenizer(num_words=VOCAB_SIZE, lower=True, split=\" \")\n",
    "tok.fit_on_texts(train_data[TEXT_COL])\n",
    "train_seq = tok.texts_to_sequences(train_data[TEXT_COL])\n",
    "train_seq = sequence.pad_sequences(train_seq, maxlen=MAX_DOC_LEN)\n",
    "test_seq = tok.texts_to_sequences(test_data[TEXT_COL])\n",
    "test_seq = sequence.pad_sequences(test_seq, maxlen=MAX_DOC_LEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the ratings to one-hot categorical labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = to_categorical(np.asarray(train_data[LABEL_COL]))\n",
    "labels = labels.astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of reviews by class in training set\n",
      "[ 5020.  4980.]\n"
     ]
    }
   ],
   "source": [
    "print('Number of reviews by class in training set')\n",
    "print(labels.sum(axis=0))\n",
    "n_classes = labels.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train word2vec on all the documents in order to initialize the word embedding. Ignore rare words (min_count=6). Use skip-gram as the training algorithm (sg=1)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\anargyri\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers\\punkt.zip.\n"
     ]
    }
   ],
   "source": [
    "import nltk \n",
    "\n",
    "nltk.download('punkt')\n",
    "\n",
    "sent_lst = []\n",
    "\n",
    "for doc in train_data[TEXT_COL]:\n",
    "    sentences = nltk.tokenize.sent_tokenize(doc)\n",
    "    for sent in sentences:\n",
    "        word_lst = [w for w in nltk.tokenize.word_tokenize(sent) if w.isalnum()]\n",
    "        sent_lst.append(word_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda\\envs\\py35\\lib\\site-packages\\gensim\\utils.py:865: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n",
      "2017-10-02 14:50:10,867 : INFO : collecting all words and their counts\n",
      "2017-10-02 14:50:10,869 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-10-02 14:50:10,901 : INFO : PROGRESS: at sentence #10000, processed 150009 words, keeping 11708 word types\n",
      "2017-10-02 14:50:10,935 : INFO : PROGRESS: at sentence #20000, processed 301138 words, keeping 17064 word types\n",
      "2017-10-02 14:50:10,969 : INFO : PROGRESS: at sentence #30000, processed 450632 words, keeping 21024 word types\n",
      "2017-10-02 14:50:11,005 : INFO : PROGRESS: at sentence #40000, processed 601648 words, keeping 24173 word types\n",
      "2017-10-02 14:50:11,032 : INFO : collected 26323 word types from a corpus of 715962 raw words and 47691 sentences\n",
      "2017-10-02 14:50:11,034 : INFO : Loading a fresh vocabulary\n",
      "2017-10-02 14:50:11,063 : INFO : min_count=6 retains 6395 unique words (24% of original 26323, drops 19928)\n",
      "2017-10-02 14:50:11,064 : INFO : min_count=6 leaves 681862 word corpus (95% of original 715962, drops 34100)\n",
      "2017-10-02 14:50:11,086 : INFO : deleting the raw counts dictionary of 26323 items\n",
      "2017-10-02 14:50:11,089 : INFO : sample=0.001 downsamples 54 most-common words\n",
      "2017-10-02 14:50:11,090 : INFO : downsampling leaves estimated 503672 word corpus (73.9% of prior 681862)\n",
      "2017-10-02 14:50:11,091 : INFO : estimated required memory for 6395 words and 100 dimensions: 8313500 bytes\n",
      "2017-10-02 14:50:11,117 : INFO : resetting layer weights\n",
      "2017-10-02 14:50:11,200 : INFO : training model with 6 workers on 6395 vocabulary and 100 features, using sg=1 hs=0 sample=0.001 negative=5 window=5\n",
      "2017-10-02 14:50:12,214 : INFO : PROGRESS: at 31.79% examples, 799476 words/s, in_qsize 11, out_qsize 5\n",
      "2017-10-02 14:50:13,230 : INFO : PROGRESS: at 67.28% examples, 839710 words/s, in_qsize 8, out_qsize 2\n",
      "2017-10-02 14:50:14,149 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-10-02 14:50:14,155 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-10-02 14:50:14,158 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-10-02 14:50:14,162 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-10-02 14:50:14,171 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-10-02 14:50:14,177 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-10-02 14:50:14,178 : INFO : training on 3579810 raw words (2518637 effective words) took 3.0s, 849402 effective words/s\n"
     ]
    }
   ],
   "source": [
    "import gensim, logging\n",
    "\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "# use skip-gram\n",
    "word2vec_model = gensim.models.Word2Vec(sentences=sent_lst, min_count=6, size=EMBEDDING_DIM, sg=1, workers=os.cpu_count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the initial embedding matrix from the output of word2vec."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 6395 word vectors.\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = {}\n",
    "\n",
    "for word in word2vec_model.wv.vocab:\n",
    "    coefs = np.asarray(word2vec_model.wv[word], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "\n",
    "print('Total %s word vectors.' % len(embeddings_index))\n",
    "\n",
    "# Initial embedding\n",
    "embedding_matrix = np.zeros((VOCAB_SIZE, EMBEDDING_DIM))\n",
    "\n",
    "for word, i in tok.word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None and i < VOCAB_SIZE:\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LSTM_DIM is the dimensionality of each LSTM output (the number of LSTM units).\n",
    "The mask_zero option determines whether masking is performed, i.e. whether the layers ignore the padded zeros in shorter documents. CNTK / Keras does not support masking yet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 50\n",
    "NUM_EPOCHS = 30\n",
    "LSTM_DIM = 100\n",
    "OPTIMIZER = SGD(lr=0.01, nesterov=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lstm_create_train(reg_param):\n",
    "    l2_reg = regularizers.l2(reg_param)\n",
    "\n",
    "    # model init\n",
    "    embedding_layer = Embedding(VOCAB_SIZE,\n",
    "                                EMBEDDING_DIM,\n",
    "                                input_length=MAX_DOC_LEN,\n",
    "                                trainable=True,\n",
    "                                mask_zero=False,\n",
    "                                embeddings_regularizer=l2_reg,\n",
    "                                weights=[embedding_matrix])\n",
    "\n",
    "    lstm_layer = LSTM(units=LSTM_DIM, kernel_regularizer=l2_reg)\n",
    "    dense_layer = Dense(n_classes, activation='softmax', kernel_regularizer=l2_reg)\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(embedding_layer)\n",
    "    model.add(Bidirectional(lstm_layer))\n",
    "    model.add(dense_layer)\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=OPTIMIZER,\n",
    "                  metrics=['acc'])\n",
    "\n",
    "    fname = \"lstm_food\"\n",
    "    history = History()\n",
    "    csv_logger = CSVLogger('./{0}_{1}.log'.format(fname, reg_param),\n",
    "                           separator=',',\n",
    "                           append=True)\n",
    "\n",
    "    t1 = time.time()\n",
    "    # model fit\n",
    "    model.fit(train_seq,\n",
    "              labels.astype('float32'),\n",
    "              batch_size=BATCH_SIZE,\n",
    "              epochs=NUM_EPOCHS,\n",
    "              callbacks=[history, csv_logger],\n",
    "              verbose=2)\n",
    "    t2 = time.time()\n",
    "\n",
    "    # save model\n",
    "    model.save('./{0}_{1}_model.h5'.format(fname, reg_param))\n",
    "    np.savetxt('./{0}_{1}_time.txt'.format(fname, reg_param), \n",
    "               [reg_param, (t2-t1) / 3600])\n",
    "    with open('./{0}_{1}_history.txt'.format(fname, reg_param), \"w\") as res_file:\n",
    "        res_file.write(str(history.history))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "83s - loss: 0.6938 - acc: 0.5176\n",
      "Epoch 2/30\n",
      "83s - loss: 0.6860 - acc: 0.5802\n",
      "Epoch 3/30\n",
      "83s - loss: 0.6775 - acc: 0.6166\n",
      "Epoch 4/30\n",
      "83s - loss: 0.6676 - acc: 0.6418\n",
      "Epoch 5/30\n",
      "83s - loss: 0.6544 - acc: 0.6618\n",
      "Epoch 6/30\n",
      "84s - loss: 0.6283 - acc: 0.6839\n",
      "Epoch 7/30\n",
      "84s - loss: 0.5386 - acc: 0.7403\n",
      "Epoch 8/30\n",
      "83s - loss: 0.5081 - acc: 0.7566\n",
      "Epoch 9/30\n",
      "83s - loss: 0.4917 - acc: 0.7671\n",
      "Epoch 10/30\n",
      "83s - loss: 0.4802 - acc: 0.7761\n",
      "Epoch 11/30\n",
      "83s - loss: 0.4732 - acc: 0.7794\n",
      "Epoch 12/30\n",
      "83s - loss: 0.4672 - acc: 0.7834\n",
      "Epoch 13/30\n",
      "83s - loss: 0.4556 - acc: 0.7879\n",
      "Epoch 14/30\n",
      "83s - loss: 0.4480 - acc: 0.7930\n",
      "Epoch 15/30\n",
      "83s - loss: 0.4443 - acc: 0.7979\n",
      "Epoch 16/30\n",
      "83s - loss: 0.4412 - acc: 0.7992\n",
      "Epoch 17/30\n",
      "83s - loss: 0.4386 - acc: 0.8002\n",
      "Epoch 18/30\n",
      "84s - loss: 0.4359 - acc: 0.7998\n",
      "Epoch 19/30\n",
      "83s - loss: 0.4277 - acc: 0.8044\n",
      "Epoch 20/30\n",
      "83s - loss: 0.4258 - acc: 0.8085\n",
      "Epoch 21/30\n",
      "83s - loss: 0.4238 - acc: 0.8087\n",
      "Epoch 22/30\n",
      "83s - loss: 0.4195 - acc: 0.8121\n",
      "Epoch 23/30\n",
      "83s - loss: 0.4134 - acc: 0.8170\n",
      "Epoch 24/30\n",
      "83s - loss: 0.4164 - acc: 0.8118\n",
      "Epoch 25/30\n",
      "83s - loss: 0.4165 - acc: 0.8101\n",
      "Epoch 26/30\n",
      "83s - loss: 0.4115 - acc: 0.8138\n",
      "Epoch 27/30\n",
      "83s - loss: 0.4048 - acc: 0.8191\n",
      "Epoch 28/30\n",
      "83s - loss: 0.4085 - acc: 0.8186\n",
      "Epoch 29/30\n",
      "83s - loss: 0.3947 - acc: 0.8270\n",
      "Epoch 30/30\n",
      "83s - loss: 0.3976 - acc: 0.8245\n"
     ]
    }
   ],
   "source": [
    "lstm_create_train(1e-7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.8274 \t AUC = 0.9012521400342406\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, roc_auc_score, roc_curve\n",
    "\n",
    "model = load_model('./lstm_food_{0}_model.h5'.format(1e-7))\n",
    "preds = model.predict(test_seq, verbose=0)\n",
    "print((\"Accuracy = {0} \\t AUC = {1}\".format(accuracy_score(test_data[LABEL_COL], preds.argmax(axis=1)), \n",
    "       roc_auc_score(test_data[LABEL_COL], preds[:, 1]))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x1a2bb2530b8>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHyBJREFUeJzt3Xt0nXWd7/H3d+/c722T3i9pSy+Uq6UUUEFucnXE4TgO\n6tEjol2dEY7H4xxgdEaXumYdXM5xlCPaqQwg6sigolOPFbxLtRTaYimUUhp6TS80aZO0uWfv/T1/\n7N1NSJM0bfPsJ8nzea2VlTyXZH9/Sft89u+5/H7m7oiIiADEwi5ARERGDoWCiIhkKRRERCRLoSAi\nIlkKBRERyVIoiIhIlkJBRESyFAoiIpKlUBARkay8sAs4VdXV1V5bWxt2GSIio8rGjRsb3b3mZPuN\nulCora1lw4YNYZchIjKqmNnuoeyn00ciIpKlUBARkSyFgoiIZCkUREQkS6EgIiJZgYWCmT1kZofM\n7KUBtpuZ3W9mdWa22cwWB1WLiIgMTZA9hUeAGwbZfiMwL/OxDPhWgLWIiMgQBPacgrs/bWa1g+xy\nC/Cop+cDXWdmVWY2xd0PBFWTiMiZSqac7kSKY109dHQn6UmmaGrvIZF0OhNJ9jd3UFIQJ5WClDvu\n6c8pP77s7GvupLwoffg9PiWyO3jmM4Dj2XXHd1hSO54r5p/0+bMzEubDa9OAvb2W6zPrTggFM1tG\nujfBzJkzc1KciIwePckUXYkUbV0J2roSdCVSHG7tprMnSdKdZOqNj33NHRxo6aAoL05dQysVRfkk\n3UmlnJQ7yczBvOFYF8mUk58XI5FM8VpDK509qVDbufwdc8d0KAyZu68EVgIsWbLET7K7iIwArV0J\n2rsTdCdSHGnrZn9zBwDdSWdfUwcFeTGOdfZwoLmT0sI8EqkUze09NLV3U1aYRyKVPlAn3Wlu7+Fo\nZw/F+XE6upPsOtxGZXE+yZRztDNx2jVWleTT3N7DnJpS4mbEzIjFjHgMYma0dPQwa0IJxfkFzBhX\nQktHD9XlhZwztYKYQTwWo6a8kIK4YWZUlxUSjxmVxfnEzTCDWMyIWfrnWeZzzIzi/DixGBjp9QAG\nmBmZRczSy7kUZijsA2b0Wp6eWSciI4S7k0g5nT1JXtp3lAMtHfQkU/QknV2NbZQW5rGzsY1kytmw\n+wilBXl0JVLsywTAqRhXkk88ZrR1JRlXkk9FcXo5L5Y+UAOUFeYxY1wJc2pKKciLMbG8iIK8GEc7\nephbU0Z+3EiknGlVxRQVxBlfUkBe3IjHjLhlPseMqpICKovzh/vXNSaEGQqrgDvN7DHgEqBF1xNE\ncsfdOdDSye7D7XQmkrxU30JXIsWaukYK82Js3X+UY10nfxcejxnJlDN9XDGdPUkunTOB/HiM9p4k\nS2vHUZgfJ5F0plQVUV1aSHFBnIJ4jNLCOKWFeRTEY9mDvoQvsFAwsx8AVwLVZlYPfB7IB3D3FcBq\n4CagDmgHbg+qFpEo6uhOcqS9mx0NrRztSLBpbxMNx7rYdbid3YfbaGrv6ff78mLpd9uXzZlAcUGc\nquJ8Fk2tAOCcqZXMnFBCftwojMcpL8rTAX2MCfLuo/efZLsDnwjq9UXGCnenoyfJniPtHG7tZl9z\nB01t3bRm3sWnz7G3U16Ux9YDR2ls7aKxtXvAn1dVkk9pQR5zaspYNKWCS+aMp6Ion+qyQmqrSygp\nGBWXGiUg+uuLhKizJ8krB4+x/fVjbDt4jN+8cohjnYnsufxkyrMH/4HkxYyUO8X5cSZXFlFelM9Z\nE8uYWlnMwinlVBUXUFtdyuSKIqaNKyaud/YyCIWCyDDrTqSoO9TKC/XNJFJOV0+SukOtHG7rpjg/\nzvZDrZQUxNm4u6nf7581oYTL51WTF4tlL7R29CSZUlnM7OoSplWVUF6Ux6SKIoryYzm/O0XGNoWC\nyBlIpdIPLP30z/t59fVjPLfzCC8fODro9yycXM6B5g6uPXsiB1o6uW7RZBbPqmLBpHJqygt1kJdQ\nKRRETuL4XTr1TR3UHWpl7WuNPPnSQSqL8zncduK5+7OnVHD5vGreOncCc2vKqCjOpyg/RmFePITq\nRU6NQkEkozuRfmp14+4mnt/dxNPbGwAGvGjb3p3k1rdMo7wojxnjS3j3BVMZX1pAXlyDD8vopVCQ\nyDnc2sXepg72N3cQM6OxtYt/f3bPCad9yovyuGjWOGonlBKPGbXVpcybWMb8SeWMLy0IqXqRYCkU\nZEw72tlDw7Eumtt7eGTtLtZsb6B5gPvzy4vyuO3iGdx8/lRqJ5RQVaIDv0SPQkFGva5EklcPtnL/\nb7ez90g7e4+0A9DWnex3/5ryQj729tksnFLB+JICivJj1FaXkq/TPiIKBRkd2roSbNjdxOstnXQl\nU/xh2yFaOnpYv+vE2zqnVRUzc3wJF86soqsnxdSqIiZWFDF7QinnTqvQ3T0ig1AoyIhztLOHtXWN\nbH+9le2HWlmzvWHAIRmuXjiRmrJCZteU8ra51Troi5whhYKEzt3ZuLuJVw4e4x9+euLsrSUFca6Y\nX8O1Z09k8cxxlBflMb60gPIijXIpMtwUCpIz7s5zO4/wyNpdvH60k+f3NFMQj9GdfPPEJVMri/jk\ntfO4uHY808eVUJCnc/0iuaJQkMC0dSXYsv8oOxtb+cFze9m0t/lN2y+ZPZ6eZIq3zq2mK5Hk6oWT\nuGBGpQZkEwmR/vfJsOhKJNm0p5kNu5vY2djGH15toOFY15v2GV9awNULJ3L3DQuYWF4UUqUiMhiF\ngpy2hmNdPL5hL6tfPMCW/W9+8GtqZRE3nz+Ft82tZvGsKmaO15DMIqOB/pfKKTnS1s231+zg20/v\nIJF6Y7rsaVXF3Lp4GlcuqOGcqZUU5WucH5HRSKEgJ7Vux2F+vvkAOxvb+GNd45u2ffGWc7h18XTK\nCvVPSWQs0P9k6Vcy5Xz3mV18e83O7CTslcX5XH/OJK5ZOIl3XzhVvQGRMUihIG/y+tFOfr31dT77\nkzeeFzh3WgX33Xo+506rDLEyEckFhYKw+3AbX//Ndp54ft8J2174/HVUFushMZGoUChE2NrXGvm7\nx19gf0tndl1NeSH/cPPZ3HjuFD00JhJBCoUIevKlAyz/3vPZ5fmTyvjCu8/lsrkTQqxKREYChUJE\n9CRT/Prl1/n8qi0c6vVQ2S8/dQXzJ5WHWJmIjCQKhTEulXLu+fFmfrixPruusjif796xlPOnV4VY\nmYiMRAqFMcjd2bL/KP/08608s+Nwdv2N507m7288m5kTSkKsTkRGMoXCGOLu/HBjPXf/aPOb1r/n\nwql85a8u0MxiInJSCoUx4oW9zXzk4eeyk9HcdN5k7r5+IbXVpSFXJiKjiUJhlPv3Z/fwmZ+8mF2e\nOb6E796xlFkTFAYicuoUCqNQe3eCjz+6gT/VvXG94LpFk/jY5XNYOnt8iJWJyGinUBhFuhJJ7nhk\nQ3ZQupjBHW+fzd9ceRbjSwtCrk5ExoJAQ8HMbgC+DsSBB939vj7bK4HvATMztfyzuz8cZE2jkbvz\n+Ia93PPjN04T/c2Vc7n7+gWapF5EhlVgoWBmceAB4J1APbDezFa5+8u9dvsE8LK7/4WZ1QDbzOz7\n7t4dVF2jzbodh7lt5brs8pULanj4IxcrDEQkEEH2FJYCde6+A8DMHgNuAXqHggPllj7ClQFHgESA\nNY0qX/3lNu7/bV12ec3dVzFjvJ4xEJHgBBkK04C9vZbrgUv67PMNYBWwHygH/trdUwHWNCq0diW4\n9v/8gYNH0wPVPXz7xVy1YGLIVYlIFIR9ofl6YBNwNTAX+JWZrXH3N034a2bLgGUAM2fOzHmRufTH\n7Y381397FoDL5kzgOx9dqtFKRSRngjza7ANm9FqenlnX2+3AE55WB+wEFvb9Qe6+0t2XuPuSmpqa\nwAoOW92hY9lA+NJ7zuUHyy5VIIhITgV5xFkPzDOz2WZWANxG+lRRb3uAawDMbBKwANgRYE0j1ub6\nZq796tMA/N118/nQpbNCrkhEoiiw00funjCzO4GnSN+S+pC7bzGz5ZntK4AvAY+Y2YuAAfe4e+OA\nP3SM+uWWgyz77kYAPn75bO68el7IFYlIVAV6TcHdVwOr+6xb0evr/cB1QdYw0n32Jy/y/Wf3AHDf\nredx29Kxfc1EREa2sC80R9rXf709GwgrP3QR150zOeSKRCTqFAoh+d663fzLr18F4LeffgdzaspC\nrkhERKEQii/8bAsP/2kXAA99ZIkCQURGDIVCDiVTzr29psZUD0FERhqFQg791Yq1PL+nmXjM+NWn\nrlAgiMiIo1DIkQd+V8fze5opiMd45Us3EItpQDsRGXn0uGwOJJIpvvLUNgA2/uO1CgQRGbEUCgGr\nO3SMsz77CwD+9sq5lBflh1yRiMjAFAoB2rK/JTt0BcCn3jk/xGpERE5O1xQCsmV/Czff/0cAPv3O\n+dx1jYauEJGRTz2FAOxqbMsGwl1Xn6VAEJFRQz2FALzvX58B4HPvWsRH3z475GpERIZOPYVh9rtt\nhzh0rIsJpQUKBBEZdRQKw6i9O8HtD68H4Psf7zvzqIjIyKdQGEZv+eKvALho1jgWTq4IuRoRkVOn\nUBhGXYkUAD9aflnIlYiInB6FwjD56Z/T009/+LJZmOmJZREZnRQKw+R//McmAM2tLCKjmkJhGHz5\nyVcAGFeSz7xJ5SFXIyJy+vScwhn6+KMb+NXLrwPw8/9+ecjViIicGfUUzsDvth3KBsKau69ialVx\nyBWJiJwZhcIZ+OwTLwLwf9//FmaMLwm5GhGRM6dQOE1r6xrZ39LJO+bX8BcXTA27HBGRYaFQOE2f\n/uELAHzlveeHXImIyPBRKJyGta81cqClk3OmVjCxoijsckREho1C4RTVN7XzgW8/C8AX3n1OyNWI\niAwvhcIp+ut/XQfAnVedxZLa8SFXIyIyvBQKp6AnmWJfcwcAn75OU2uKyNijUDgF9/x4MwD/TeMb\nicgYpVAYoo7uJE88nx707rM3Lwq5GhGRYCgUhuhrv3kVgP91/QIK8vRrE5GxKdCjm5ndYGbbzKzO\nzO4dYJ8rzWyTmW0xsz8EWc+ZqG9KX0u4Q1NsisgYFtiAeGYWBx4A3gnUA+vNbJW7v9xrnyrgm8AN\n7r7HzCYGVc+Z+s3W9BhHRfnxkCsREQlOkD2FpUCdu+9w927gMeCWPvt8AHjC3fcAuPuhAOs5bS/t\na6GzJ8WSWePCLkVEJFBBhsI0YG+v5frMut7mA+PM7PdmttHMPtzfDzKzZWa2wcw2NDQ0BFTuwO76\nwZ8B+PR1C3L+2iIiuRT2FdM84CLgZuB64B/N7IQHANx9pbsvcfclNTU1OS1wf3MHOxvbKM6Pc9nc\nCTl9bRGRXAtykp19wIxey9Mz63qrBw67exvQZmZPAxcArwZY1yl59JndAHz9tgtDrkREJHhB9hTW\nA/PMbLaZFQC3Aav67POfwNvNLM/MSoBLgK0B1nRK3J3fb0tf5rj27EkhVyMiErzAegrunjCzO4Gn\ngDjwkLtvMbPlme0r3H2rmT0JbAZSwIPu/lJQNZ2qjz6ynlcOHmPh5HJiMT3BLCJjX6BzNLv7amB1\nn3Ur+ix/BfhKkHWcrt9tS1/UfvSjS0OuREQkN8K+0DxiPb+nCYBrFk7UnAkiEhkKhQH84sUDACy7\nYk7IlYiI5I5CoR/uzrfX7ARg6WzNmSAi0aFQ6MePNtYDcMGMKg2RLSKRolDox7odRwD4l/ddEHIl\nIiK5pVDoR2F++tcyp6Ys5EpERHLrlEPBzGJm9sEgihkp/mP9XqZU6o4jEYmeAUPBzCrM7O/N7Btm\ndp2l3QXsAN6XuxJzqzuRIplyDrR0hl2KiEjODfbw2neBJuAZ4GPAZwAD3uPum3JQWyiOD2vx8cs1\nmY6IRM9goTDH3c8DMLMHgQPATHcf02+hv/Cz9BxAty6eHnIlIiK5N9g1hZ7jX7h7Eqgf64HQ2ZNk\nX3N62s2zp1SEXI2ISO4N1lO4wMyOkj5lBFDca9ndfcwdNVe9sB+Aj7y1NtxCRERCMmAouHvkJiP+\nztpdANz+ttpQ6xARCcuAoWBmRcBy4CzSQ1s/5O6JXBUWhi37jwIwa0JpyJWIiIRjsNNH3yF9XWEN\ncBNwDvDJXBQVlsrifGqrFQgiEl2DhcKiXncf/RvwXG5KCoe709LRw5JZ48IuRUQkNEO9+2hMnzYC\nePX1ViD98JqISFQN1lO4MHO3EaTvOBrTdx+1dqUz8KqFNSFXIiISnsFC4QV3f0vOKglZT9IBKMqL\n3E1XIiJZg50+8pxVMQK8tK8FgFhM8yeISHQN1lOYaGb/c6CN7v7VAOoJzbodhwFYMKk85EpERMIz\nWCjEgTLeeKJ5TDv+jMK40oKQKxERCc9goXDA3b+Ys0pClEimONDSySKNdyQiETfYNYVI9BAAntuV\nnn7zhnMnh1yJiEi4BguFa3JWRchea2gD4MoFuh1VRKJtwFBw9yO5LCRMuxvToTCtqjjkSkREwnXK\nczSPRa8f6wJgQllhyJWIiIRLoUB63CMREVEoALD2tcPMGK9TRyIikQ+Fzp4kR9q6KdTwFiIiwYaC\nmd1gZtvMrM7M7h1kv4vNLGFm7w2ynv7szFxkvnxeda5fWkRkxAksFMwsDjwA3AgsAt5vZosG2O/L\nwC+DqmUwWw+kn2R+61yFgohIkD2FpUCdu+9w927gMeCWfva7C/gxcCjAWgb05EsHAZg/qSyMlxcR\nGVGCDIVpwN5ey/WZdVlmNg34S+BbAdYxoENHO/nly68DMHN8SRgliIiMKGFfaP4acI+7DzrdmZkt\nM7MNZrahoaFh2F78G7+rA+B/33oeZpEZ1UNEZECDDYh3pvYBM3otT8+s620J8FjmgFwN3GRmCXf/\nae+d3H0lsBJgyZIlw/ZQwZ4j7QD8l8XTh+tHioiMakGGwnpgnpnNJh0GtwEf6L2Du88+/rWZPQL8\nv76BEKRXDx6jsjifgrywO0wiIiNDYEdDd08AdwJPAVuBx919i5ktN7PlQb3uqThwtFPjHYmI9BJk\nTwF3Xw2s7rNuxQD7fiTIWvpKpRx3WDyrKpcvKyIyokX2vMmhzCB4pYWB5qKIyKgS2VDYfugYALPG\nl4ZciYjIyBHZUGjvTgJQW63nE0REjotsKOxr6gBgYrnmUBAROS6yodDc0QNARXF+yJWIiIwckQ2F\n53YeBmB8SUHIlYiIjByRDYXN9S0A5MUj+ysQETlBJI+IqZTT3p3k4tpxYZciIjKiRDIUnt15BIBz\np1WGXImIyMgSyVBoaE0/uHbrWzQQnohIb5EMhb2Z0VGLCzQvs4hIb5EMhYbMEBc1ZXpGQUSkt0iG\nQn3mwbWyIo17JCLSW0RDoZ3K4nziMc22JiLSWyRDoa07QY2GtxAROUEkQyFuxlk1ZWGXISIy4kQy\nFBqOdVGiO49ERE4QyVBo605mB8QTEZE3RDIU4jFjdrUm1xER6SuSoWBAUX4kmy4iMqjIHRndnUTK\niZtuRxUR6StyoXB8Gs7jn0VE5A2RC4W2rgQAkyuLQq5ERGTkiVwotGZCoaJI03CKiPQVuVDYfbg9\n7BJEREasyIXCKwePAbBoakXIlYiIjDyRC4XjzpqoYS5ERPqKXCjsamwDID8euaaLiJxU5I6MBXnp\nJmvYbBGRE0UuFLYeOEp1WUHYZYiIjEiRC4WqkgI9uCYiMoBAQ8HMbjCzbWZWZ2b39rP9g2a22cxe\nNLO1ZnZBkPUApNyZq7kURET6FVgomFkceAC4EVgEvN/MFvXZbSfwDnc/D/gSsDKoeo5LuaPLCSIi\n/Quyp7AUqHP3He7eDTwG3NJ7B3df6+5NmcV1wPQA6wEgmXJiSgURkX4FGQrTgL29lusz6wZyB/CL\n/jaY2TIz22BmGxoaGs6oqKb2bmIaIVVEpF8j4kKzmV1FOhTu6W+7u6909yXuvqSmpuaMXquprYcW\nzbomItKvvAB/9j5gRq/l6Zl1b2Jm5wMPAje6++EA6wGgrDCPmvLCoF9GRGRUCrKnsB6YZ2azzawA\nuA1Y1XsHM5sJPAF8yN1fDbCWrKQ7lcUaIVVEpD+BhYK7J4A7gaeArcDj7r7FzJab2fLMbp8DJgDf\nNLNNZrYhqHqO23ukXU8zi4gMIMjTR7j7amB1n3Uren39MeBjQdbQV1ciRbOuKYiI9GtEXGjOlcbW\nLgBmTygJuRIRkZEpUqFwsKUTgLMmlYdciYjIyBSpUNh7JD3r2uQKzc8sItKfSIWCZR5am1KpUBAR\n6U+kQqE7mQLemFNBRETeLFJHx92ZWdcKNOuaiEi/InV03NuUvqagJ5pFRPoXqVDoTqRPH5UUxEOu\nRERkZIpUKGw/1MqkisLsBWcREXmzSIVCVUk+RfnqJYiIDCRSoZBMOZPKdTuqiMhAIhUKqRTEItVi\nEZFTE6lDZFciqRFSRUQGEalQqDvUSk/Cwy5DRGTEilQojC8rIC+unoKIyEAiFQqpFEytKg67DBGR\nEStSobCvuYO4nlEQERlQpEIBoLUrEXYJIiIjVqRCwQzm1JSGXYaIyIgVmVBIpRx3dEuqiMggIhMK\niVT6VtQ8hYKIyIAiEwotHT0AHO3UNQURkYFEJhSSmZ5C7QRdUxARGUh0QsHToaBJ10REBhaZQ2Qq\n01OI6TkFEZEBRScUsj0FhYKIyEAiEwrN7ekLzcfvQhIRkRNFJhSOh0FZYV7IlYiIjFyRCYXjp4/K\nixQKIiIDiUwoHL8lVQPiiYgMLNBQMLMbzGybmdWZ2b39bDczuz+zfbOZLQ6qluM9hZguNIuIDCiw\nUDCzOPAAcCOwCHi/mS3qs9uNwLzMxzLgW0HVc6StG9AtqSIigwmyp7AUqHP3He7eDTwG3NJnn1uA\nRz1tHVBlZlOCKCbTUUAdBRGRgQUZCtOAvb2W6zPrTnWfYXG8h1BRnB/EjxcRGRNGxYVmM1tmZhvM\nbENDQ8Np/YzJlUXcdN5k3ZIqIjKIII+Q+4AZvZanZ9ad6j64+0pgJcCSJUtO6+mzi2aN46JZF53O\nt4qIREaQPYX1wDwzm21mBcBtwKo++6wCPpy5C+lSoMXdDwRYk4iIDCKwnoK7J8zsTuApIA485O5b\nzGx5ZvsKYDVwE1AHtAO3B1WPiIicXKAn2N19NekDf+91K3p97cAngqxBRESGblRcaBYRkdxQKIiI\nSJZCQUREshQKIiKSpVAQEZEscx9dM5GZWQOw+zS/vRpoHMZyRgO1ORrU5mg4kzbPcveak+006kLh\nTJjZBndfEnYduaQ2R4PaHA25aLNOH4mISJZCQUREsqIWCivDLiAEanM0qM3REHibI3VNQUREBhe1\nnoKIiAxiTIaCmd1gZtvMrM7M7u1nu5nZ/Zntm81scRh1DqchtPmDmba+aGZrzeyCMOocTidrc6/9\nLjazhJm9N5f1BWEobTazK81sk5ltMbM/5LrG4TaEf9uVZvYzM3sh0+ZRPdqymT1kZofM7KUBtgd7\n/HL3MfVBepju14A5QAHwArCozz43Ab8ADLgUeDbsunPQ5rcC4zJf3xiFNvfa77ekR+t9b9h15+Dv\nXAW8DMzMLE8Mu+4ctPkzwJczX9cAR4CCsGs/gzZfASwGXhpge6DHr7HYU1gK1Ln7DnfvBh4Dbumz\nzy3Ao562Dqgysym5LnQYnbTN7r7W3Zsyi+tIz3I3mg3l7wxwF/Bj4FAuiwvIUNr8AeAJd98D4O6j\nvd1DabMD5WZmQBnpUEjktszh4+5Pk27DQAI9fo3FUJgG7O21XJ9Zd6r7jCan2p47SL/TGM1O2mYz\nmwb8JfCtHNYVpKH8necD48zs92a20cw+nLPqgjGUNn8DOBvYD7wIfNLdU7kpLxSBHr80i33EmNlV\npEPh7WHXkgNfA+5x91T6TWQk5AEXAdcAxcAzZrbO3V8Nt6xAXQ9sAq4G5gK/MrM17n403LJGp7EY\nCvuAGb2Wp2fWneo+o8mQ2mNm5wMPAje6++Ec1RaUobR5CfBYJhCqgZvMLOHuP81NicNuKG2uBw67\nexvQZmZPAxcAozUUhtLm24H7PH3Cvc7MdgILgedyU2LOBXr8Gounj9YD88xstpkVALcBq/rsswr4\ncOYq/qVAi7sfyHWhw+ikbTazmcATwIfGyLvGk7bZ3We7e6271wI/Av52FAcCDO3f9n8CbzezPDMr\nAS4Btua4zuE0lDbvId0zwswmAQuAHTmtMrcCPX6NuZ6CuyfM7E7gKdJ3Ljzk7lvMbHlm+wrSd6Lc\nBNQB7aTfaYxaQ2zz54AJwDcz75wTPooHExtim8eUobTZ3bea2ZPAZiAFPOju/d7aOBoM8e/8JeAR\nM3uR9B0597j7qB091cx+AFwJVJtZPfB5IB9yc/zSE80iIpI1Fk8fiYjIaVIoiIhIlkJBRESyFAoi\nIpKlUBARkSyFgsgQmVkyM/ro8Y/azIikLZnlrWb2+cy+vde/Ymb/HHb9IkMx5p5TEAlQh7tf2HuF\nmdUCa9z9XWZWCmwys59lNh9fXwz82cx+4u5/ym3JIqdGPQWRYZIZWmIjcFaf9R2kx+YZzYMuSkQo\nFESGrrjXqaOf9N1oZhNIj2+/pc/6ccA84OnclCly+nT6SGToTjh9lHG5mf2Z9LAS92WGYbgys/4F\n0oHwNXc/mMNaRU6LQkHkzK1x93cNtN7MZgPrzOxxd9+U6+JEToVOH4kEzN13AvcB94Rdi8jJKBRE\ncmMFcEXmbiWREUujpIqISJZ6CiIikqVQEBGRLIWCiIhkKRRERCRLoSAiIlkKBRERyVIoiIhIlkJB\nRESy/j8G2dv1PBbZBAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a2f52dd0b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, _ = roc_curve(test_data[LABEL_COL], preds[:, 1])\n",
    "plot(fpr, tpr)\n",
    "xlabel('FPR')\n",
    "ylabel('TPR')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}