4.Assignment on NB(Revised).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from sklearn.metrics import accuracy_score\n",
    "from collections import Counter\n",
    "from sklearn.metrics import accuracy_score\n",
    "import warnings\n",
    "from sklearn.metrics import f1_score\n",
    "from scipy import sparse\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=pd.read_csv('train')\n",
    "cv=pd.read_csv('cv')\n",
    "test=pd.read_csv('test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x=train['CleanedText']\n",
    "cv_x=cv['CleanedText']\n",
    "test_x=test['CleanedText']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y=train['Score']\n",
    "cv_y=cv['Score']\n",
    "test_y=test['Score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "uni_gram = CountVectorizer()\n",
    "X_train = uni_gram.fit_transform(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading BoW data\n",
    "\n",
    "train_bow=sparse.load_npz(\"train_bow.npz\")\n",
    "cv_bow=sparse.load_npz(\"cv_bow.npz\")\n",
    "test_bow=sparse.load_npz(\"test_bow.npz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def BernouliNB(train_tfidf,train_y,cv_tfidf,cv_y):\n",
    "    error_list=[]\n",
    "    best_alpha=0;max_roc_auc=-1\n",
    "    alpha=[100,50,10,5,1,0.5,0.1,0.05,0.01,0.005,0.001,0.0005,0.0001] \n",
    "    for i in alpha[-20:]:\n",
    "        bnb = BernoulliNB(alpha=i)\n",
    "        bnb.fit(train_tfidf,train_y)\n",
    "        pred = bnb.predict(cv_tfidf)\n",
    "        #acc = accuracy_score(cv_y, pred, normalize=True) * float(100)\n",
    "        roc_auc=(roc_auc_score(cv_y,pred)*100)\n",
    "        error_list.append(roc_auc-1)\n",
    "        if(max_roc_auc<roc_auc):\n",
    "            max_roc_auc=roc_auc\n",
    "            best_alpha=i\n",
    "    print(f\"\\n Best alpha Value {best_alpha} with highest roc_auc Score is {roc_auc}\")\n",
    "    sns.set_style(\"darkgrid\")\n",
    "    plt.plot(alpha,error_list)\n",
    "    plt.title(r'Accuracy v/s $\\alpha$')\n",
    "    plt.xlabel(r\"alpha values\",fontsize=12)\n",
    "    plt.ylabel(\"accuracy\",fontsize=12)\n",
    "    plt.show()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def testBernouli(train,train_y,test_bow,test_y,best_alpha):\n",
    "    bnb = BernoulliNB(best_alpha)\n",
    "    bnb.fit(train,train_y)\n",
    "    pred = bnb.predict(test_bow)\n",
    "    roc_auc=(roc_auc_score(test_y,pred)*100)\n",
    "    print(f\"best roc auc score is {roc_auc}\")\n",
    "    print(\"Confusion Matrix of test set:\\n [ [TN FN]\\n [FP TP] ]\\n\")\n",
    "\n",
    "    df_cm = confusion_matrix(test_y,pred)\n",
    "    sns.set(font_scale=1.4) #for label size                     \n",
    "    sns.heatmap(df_cm, annot=True,annot_kws={\"size\": 16}, fmt='g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Best alpha Value 1 with highest roc_auc Score is 49.64246328141477\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEVCAYAAAAVeRmFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XtcVNXeP/DPZoYBhqt4BK+pmAqlmFiaCop20qw0L2RocjKfHrVDmmaioagn8cLp8WWaL007j9YRL5WgPv46pzIveQnNzEgNyyw1b6goRxguc9nr9wcygjLOALOBmf15v169ZIZ9+a7R+LDW2nttSQghQEREVAWP+i6AiIgaLoYEERHZxJAgIiKbGBJERGQTQ4KIiGxiSBARkU0MCSIisokhQURENjEkyC2YTCZER0fjlVdeqe9SFDV58mScPn26vssgFdHWdwFEzrBz506Eh4fjxIkTOHPmDNq1a1ffJTmd0WjE+fPn0b59e4f3MZvNWL16NTIyMmAwGDB79mxcuXIFZrMZr776qoLVkrtgT4LcwqZNm/DEE0/g6aefxkcffVTpe1u2bMEzzzyDwYMH4y9/+QsuX75s8/3Dhw/j2Wefte5b/vrw4cMYMmQI4uPjMXjwYBiNRqSmpuL555/H008/jUGDBuHo0aP3Pefs2bOxdOlS6zbbt29HYmJipVqnTZuGtWvXWl9v3LgRU6ZMAQB888036NmzJwDAYDBg8uTJeO655zBs2DDMnj0bsizf87m8++67OHLkCLZv345FixZh5cqV2Lp1KxISEmr6UZPaCCIXd/r0afHwww+LGzduiOzsbBEZGSlu3LghhBAiJydH9OjRQ1y6dEkIIcS6detESkqKzfcPHToknnnmGeuxy18fOnRIhIeHiwsXLgghhPj+++/FpEmThMViEUIIsXr1ajFhwoT7nvOnn34SvXv3FiaTSQghxOjRo8W+ffsqtSUrK0s8++yz1tdxcXHi4MGDQgghZs2aJY4cOSKEEGLr1q1i3LhxQgghzGazmDVrljh79mylYxUUFIhOnTqJ8+fPCyGEuH79uujQoYNYtWpVzT9sUh0ON5HL27RpE/r164dGjRqhUaNGaNmyJT755BNMmDABWVlZiI6ORrNmzQAAY8eOBQCsW7euyvcPHz5s8zzNmjVDixYtAABdu3ZFYGAgNm/ejD/++AOHDx+Gr68vANg8JwC0bNkSe/fuRdu2bXH16lVER0dXOkePHj1QWlqK48ePw8fHBzdu3EDPnj0hhEB2djbefvttAEC3bt2wdOlSJCQkoFevXnjppZfQunXrSsc6dOgQ2rRpg1atWgEom7fx9/fHmDFjqvsRk4pxuIlcWlFREbZv346jR4+if//+6N+/P65du4b09HSYTCZoNBpIkmTdvqSkBGfOnLH5viRJEBUWRjaZTNav9Xq99eu9e/diwoQJAIAnnngCo0aNsn7P1rEB4MUXX0RGRga2bNmCkSNHVtoOACRJQlxcHLZv346MjAzExcVBkiQcO3YMnTp1godH2f+yrVq1ws6dOzF+/HgUFhbi5Zdfxu7duysdKzc3FyEhIdbXH3/8MUJDQ+Hn51eNT5jUjiFBLm3Hjh0ICgrC/v37sXv3buzevRtfffUVioqK8Pnnn6NHjx7IysrC1atXAQCbN2/GO++8Y/P94OBgXLp0CXl5eRBC4LPPPqvyvAcPHkS/fv0wevRodOrUCV999RUsFgsA2Dw2AAwcOBA5OTn44osvMGLEiCqPPWzYMOzevRtffPEFhg8fDgDYtWsX/vznP1u32bhxI9566y1ER0dj+vTpiI6Oxk8//VTpOE2bNsWpU6dw9epVZGdnY/v27cjLy4PRaKzpx00qxOEmcmmbNm3Cyy+/DI1GY30vICAACQkJ+PDDD5GRkYHp06dbL41t0qQJFi5ciNDQUJvvx8fHY8SIEWjSpAliY2Nx/Pjxe84bHx+PadOmYfDgwTCbzejduze+/PJLyLKMjh07VnlsANDpdBg4cCCuX7+O4ODgKtvUpEkTPPTQQzCbzQgNDQVQNmk9adIk6zZDhw7Ft99+i6effho+Pj5o1qzZPZPRMTEx6N27N55++mkEBgZixYoVeOedd/DSSy9h06ZNNf3ISWUkIfjQIaK6UlRUhDFjxmDOnDl45JFH6rscIrs43ERUR/bv34/Y2FjExMQwIMhlsCdBREQ2sSdBREQ2MSSIiMgml7+6SZZlWCw1GzHTaKQa7+uq2GZ1YJvdX23b6+mpsb8RFA6JvLw8DB8+HGvXrkVJSQnmzp0LnU6HiIgIzJo1y3pjUEVnzpzByJEj8c0338DLy8vuOSwWgfz8ohrVFxSkr/G+roptVge22f3Vtr1Nmvg7tJ1iw00mkwlz5syBt7c3ACAlJQXJycnYuHEj/Pz8sGPHjnv2KSwsRFpaGnQ6nVJlERFRNSgWEmlpaYiPj7cuC5Cbm4uoqCgAQFRUVKUVMwFACIGUlBS88cYb8PHxUaosIiKqBkWGmzIzMxEcHIyYmBisWbMGQNlaM99++y26d++OPXv2oLi4uNI+K1asQN++fREeHl6tc2k0EoKC9PY3rHJfjxrv66rYZnVgm91fXbVXkfskXnzxRUiSBEmSkJOTgzZt2iApKQmrV6+GRqNB586dUVBQgOTkZOs+Tz75JJo2bQoA+OGHHxAZGYkNGzbYPZfJZOGcRDWwzerANru/upqTUKQnUfGHe0JCAubNm4d9+/ZZ18aZP38++vTpU2mfnTt3Wr/u379/pQevEBFR/aiz+yRat26N8ePHIz4+Hn5+fujbty8AYNy4cVyVkoiogXL5ZTk43FQ9bLM6sM3uz+UvgXUlZ64b8PWveXDxvCQicjqXv+PaGT7IOoddv1xHdFgwZj3ZHn/ys38THxGRGrAnAaCgxIxAby2OnM/HCx8dxZenrrJXQUQEhgQAoNhkQXioH9ITotC6kQ9mfXYKyf8vB/lFJvs7ExG5MYYEAIPRAr1OizbBeqyJfwSJ0W2w99c8vPDRd/j61+v1XR4RUb1hSKCsJ6H3LPsotB4SxvZ4AOvHRKGJnxfe3P4T5v37FApKzPVcJRFR3WNIACi63ZOo6MEmvlg3+hG88vgD+DznKuI/+g5ZZ2/UU4VERPWDIQGgyGSBTxVrq3tqPDChdxusHd0Vvl5aTM44gUU7T8NgZK+CiNRB9SFhssgwWQT0OtsfxUNN/bF+TBQSHm2JrT9exuh/fo9jF/5Th1USEdUP1YdEkdECAPcMN93NS+uByX3D8EF8F3hIwOSM49Z9iYjclepDoth0OyQ8HfsourQIxPT+D6LELOPklVtKlkZEVO9UHxJFJsd6EhV1bhYACUD2RYYEEbk3hkT5cJODDwUHAH9vLcL+pEf2JYYEEbk3hoR1TsLxkACALs0DcfzSLVhkLt9BRO6LIVGDngQAdGkRAIPRgt/yDEqURUTUIDAkTDXrSUQ2DwAA/MghJyJyY6oPifKrm3yqGRItAr3R2FfHyWsicmuqD4maDjdJkoQuzQM4eU1Ebo0hYbRAAuDt4H0SFXVpEYBL/ynB9cJS5xdGRNQAMCRur9vkIUnV3rd8XoK9CSJyVwwJo6Xak9blOob4wUvrwXkJInJbqg+JYlPNQ8JT44GHmvqzJ0FEbkv1IWEwWqo9aV1Rl+YB+PlqIUpMXOyPiNyP6kOi2GSp9uWvFXVpEQCLLHDySoETqyIiahhUHxJFtexJdG7Gm+qIyH0xJGoxcQ0AgT6eaNtYz8lrInJLqg+JYlPtehJA2bzEj5duQRZc7I+I3IvqQ8JQy54EUHa/REGpGb/nFTmpKiKihkHVISGEqPXENVD2tDqAN9URkftRdUiUmGTIAvCt5XBTqyBvNPLxxI8X/+OkyoiIGgZVh0SR0Qyg+ivA3k2SJHRpwcX+iMj9qDokDDVcAbYqkc0DcCG/BHkGY62PRUTUUKg6JMqXCa9tTwK4My/B+yWIyJ2oOiTMFhkAoPWo/gqwdwsP8YNOI/F+CSJyK+oOCbnsvgaNE0JCpy1b7O/HS5y8JiL3oeqQKL/5TVuDZ0lUJbJ5IHJyudgfEbkPRUMiLy8Pffv2xZkzZ3Dy5EnExcVh9OjRmD9/PmRZrrRtQUEBJk6ciDFjxuCFF17AsWPHlCwNwJ2ehIeTPoXI5gEwywI5uYXOOSARUT1TLCRMJhPmzJkDb29vAEBKSgqSk5OxceNG+Pn5YceOHZW2X7duHR5//HGkp6dj0aJFePvtt5UqzUp24nATULY8BwBk834JInITioVEWloa4uPjERISAgDIzc1FVFQUACAqKgpHjx6ttP3YsWMRHx8PALBYLPDy8lKqNCvrnISThpuC9J5o3ciH90sQkdvQKnHQzMxMBAcHIyYmBmvWrAEAtGrVCt9++y26d++OPXv2oLi4uNI+AQFlv4Vfu3YN06dPR3JyskPn0mgkBAXpa1ZoXlkNgQE+NT/GXR5rG4yvcq4iIMAHHk7qoTiTRuPhtLa6CrZZHdTW5rpqryIhkZGRAUmSkJWVhZycHMyYMQNJSUlYvXo1/vGPf6Bz587Q6XT37Pfzzz/jjTfeQFJSErp37+7QuSwWgfz8mi2sZzSXTTAXGUprfIy7RfzJF1uKTcj+PQ9tGze8f7BBQXqntdVVsM3qoLY217a9TZr4O7SdIiGxYcMG69cJCQmYN28e9u3bh4ULFyI0NBTz589Hnz59Ku3z66+/4vXXX8e7776L8PBwJcq6h8Xi3DkJAIhsUf4Qov80yJAgIqqOOrsEtnXr1hg/fjzi4+Ph5+eHvn37AgDGjRsHo9GIJUuWwGg0YsGCBUhISMCrr76qeE0W4dw5CQBo3cgHQT6evKmOiNyCIj2JitavXw8AaNeuHfr373/P99euXQsAWLVqldKl3MPi5KubgLLF/iKbc7E/InIPqr6ZrjwknD2/HNk8AOdvFuNmERf7IyLXxpCAc3sSwJ37JbjYHxG5OnWHRPmyHE4OiYim/vDkYn9E5AbUHRLW4SbnhoSX1gPhIf6clyAil8eQgPOHmwCgS4sA5OQWoNQs29+YiKiBYkjAuZfAluvSPAAmi8Cp3AKnH5uIqK6oOiSc+TyJu925qY5DTkTkulQdEuXPk3DWUuEVBet1eKCRDyevicilqTokzBblhpsAoHPzAPx46RbE7TAiInI1qg6J8p6EEsNNQNm8xM1iE87fLLa/MRFRA6TqkFByTgIou8IJAC+FJSKXpeqQkGUBCc6/T6Jcm2A9Ary1+JHzEkTkolQdEhZZKPpgIA/rYn98nCkRuSZ1h4QQTl+S426RzQNw9kYx8otNip6HiEgJ6g4JWTh9Bdi7lc9LHOe8BBG5IFWHhFkWik1al3so1B8aD4mT10TkklQdErIsFLtHopy3pwbhIX748SLnJYjI9ag6JOqiJwGUDTn9lFsIk4WL/RGRa1F1SMhCKHb5a0Vdmgeg1CzjVG6h4uciInImVYdEXfUkIlsEAuBNdUTkelQdEmVzEsqf50++OrQI9EY25yWIyMWoOiTqqicBlM1LcLE/InI1qg4JuS5DonkAbhSZcCG/pE7OR0TkDKoOCbNcNxPXwJ15CT6EiIhciapDwlKHPYmwxnr4eWm4jhMRuRR1h4RQ/ma6ch6ShM7NAvikOiJyKeoOiTrsSQBlk9e/5RXhVgkX+yMi16D6kKirOQkA6NK8bF7i+KWCOjsnEVFtqD4ktHX4CTzczB8aCZyXICKXoe6QEMo+dOhuPp4adAjx47wEEbkMdYdEHawCe7cuLQJx8koBSs1c7I+IGj5tfRdQn8yygK4ux5sAPNIiAJu/v4joZQfgrfWAn5cWvjoN/Ly08PPSVH6t08LXSwM/Xdn3fL20d31fAy+tB6Q6DjoiUg9Vh4QsK//40rv1adcYb/35QeQVmWAotaDQaIah1IzCUgsMRjOuFhhvv2dBkcli93haD6lCyNwOk9uhYg0X3Z0ACgn2hWQylwXO7f30Ok2dTuATketQdUiY62G4yVPjgeFdmju0rUUWMBjvBEhhqQWFpWZriJR9XfanofzPUjMu3yopC57b78l2louSAOh1mnt7NZV6MvcGzt09Ha1G1aOXRG5J1SEhy3U7cV1dGg8JAd6eCPD2rPExhBAoNsnWkIFOiyt5hTZDpvzrm7fXmSp/7cgcipfWo3KvRqep1GO537Ba+XCaN4fPiBoUh0Ji0qRJGDVqFHr16qV0PXWqPnoSdU2SJOh1Guh1GjTxA4KC9Mj311X7OEazfP9eTYX37gSOBdcMRTDcfs9gtD98pvGQrOFSce6lcshUNax2p1ej12nq9CZJInfmUEg8+eSTWLlyJf72t79h5MiRGDFiBIKCghw6QV5eHoYPH461a9eipKQEc+fOhU6nQ0REBGbNmgUPjztDFCUlJZg+fTry8vLg6+uLtLQ0BAcH16xlDpCFAEdIHKPTekCn1aGRvubHsMgCRcbKIWMNlKoC6Pb3cwtKcSav6PbcjRkWB1Zb13tqrL2TIL0OXhrJ2mOxOaxWsVej09b5RQ1EDZFDITFkyBAMGTIEZ86cQUZGBuLi4tC1a1ckJCQgMjLS5n4mkwlz5syBt7c3ACAlJQWzZ89GVFQUli5dih07duC5556zbr9p0yZ06NABkyZNwmeffYaVK1di9uzZtWyibXX5PAkq6yX4e2vh713zUU4hBErNsrWnYg0Vo/meXkz56xJZIN9gxKX/lFi/78jwmU4jWXsuvhWGzioOod1vHsfHU4P66qhqSkwoLDXXz8nridra7FNHl9E7/H+rLMs4d+4czp49C4vFgsaNG2PevHmIjY3F5MmTq9wnLS0N8fHxWLNmDQAgNzcXUVFRAICoqCjs2rWrUkgcPXoUr7zyCgCgT58+WLlyZY0b5lib6nZZDqo9SZLg7amBt6cGf/JzbJ+gID3y84sqvWe2yNZwqThcZrgdNraG1f4oMlrfKzJawEdIUX0J8Nbis/E94O2pUfQ8DoXE0qVLkZmZiVatWmH06NFYtmwZPD09UVRUhH79+lUZEpmZmQgODkZMTIw1JFq1aoVvv/0W3bt3x549e1BcXFxpn8LCQvj7+wMAfH19UVBgf40jjUZCUFDNxkAsQkDv7Vnj/V2RRuOhqvYCttv8p1oeV5bF7Z6JCQUlZeFSUHL7v1ITSkz1d8OkJEmqewqi2trcNNAbIY39FL/4xqGQuHHjBj744AOEh4dXel+v12PJkiVV7pORkQFJkpCVlYWcnBzMmDEDSUlJWL16Nf7xj3+gc+fO0OkqT6D6+fnBYDAAAAwGAwICAuzWZrGIe35LdJTZImA2WWq8vyuq6rdqd6d0m30A+HhpEOKlAQK8FDtPdfDv2f3Vtr1Nmvg7tJ1DM3OJiYnYvHkzAOC3337DX//6V1y7dg0AEB0dXeU+GzZsQHp6OtavX4+IiAikpaXh5MmTWLhwIdasWYP8/Hz07t270j5RUVH4+uuvAQD79u1Dt27dHGpETZVNXHO4iYjIFodCYubMmQgLCwMAtGjRAt27d0dycnK1T9a6dWuMHz8e8fHx8PPzQ9++fQEA48aNg9FoxKhRo3D69GmMGjUKH3/8MV577bVqn6M6OHFNRHR/knBgEO+5557D9u3bK703dOhQbNu2TbHCHGWqxXBR7HsH8Vznppga287JVTVcauuSA2yzWqitzQ1quMlisSA3N9f6+vr1624xQWSuh7WbiIhciUMT12PHjsXQoUMRExMDSZLwzTffICkpSenaFCcLXgJLRHQ/DoVEXFwcOnXqhEOHDkGj0eC//uu/0KFDB6VrUxznJIiI7s/hm+maNm2KgQMHQggBi8WCgwcP3nN1kiuRhYAQcPu1m4iIasOhkFi2bJn1hjitVguj0YgHH3wQO3bsULQ4Jcm3189mT4KIyDaHJq63b9+OPXv2YODAgfjiiy+waNEiPPjgg0rXpijz7ZBgRhAR2eZQSAQHByMkJARhYWE4deoUhg4dil9++UXp2hRV/iAe9iSIiGxzKCS0Wi3Onz+PsLAwfPfddzCbzSgtLVW6NkVZONxERGSXQyExceJEpKSkIDY2Fjt37kRsbCwef/xxpWtTlOX2fR6cuCYiss2hiWuz2YyPPvoIALBt2zacO3cOHTt2VLQwpbEnQURkn0M9iaVLl1q/9vHxQXh4uMs/h1i+3ZNoyM+4JiKqbw71JDp06IBVq1bh0UcfhV5/Z13+hx9+WLHClFbek9C6eNgRESnJoZDIzs5GdnY2Pv30U+t7kiRh165dihWmNOslsHyMMRGRTQ6FxO7du5Wuo87xElgiIvscCol169ZV+f7LL7/s1GLqknXimsNNREQ2ORQSFW+cMxqNOHLkCHr27KlYUXXBegksexJERDY5FBKLFi2q9Do3NxezZs1SpKC6YrEuy8GQICKypUbTtqGhobh48aKza6lTMnsSRER2VXtOQgiBEydOoHHjxooVVRd4Mx0RkX3VnpMAgGbNmrn8k+nuTFzXcyFERA2Yw3MSR44cwWOPPYb8/Hx89913aNq0qdK1KYoT10RE9jm8LMfy5csBACUlJVizZg1WrlypaGFK48Q1EZF9DoXErl27sHbtWgBljzFNT0/Hv/71L0ULU5osl/2pZU+CiMgmh0LCZDLB09PT+trT09PlF/gzC/YkiIjscWhOIioqCtOmTUNcXBwkScK2bdvQpUsXpWtTFJ9xTURkn0MhkZKSguXLl2PRokXQarXo1asXEhMTla5NUVyWg4jIPodCQq/X44knnsDMmTOtVzf5+PgoXZuieDMdEZF9qr26iUuFExHZp96rm8qXCudwExGRTaq9uonLchAR2Vejq5u2bt3q8lc3MSSIiOyr1tVNixcvhkajQa9evfDaa68pXZuirMtyuHiPiIhISQ4NN/388884e/YsAgMD4evri2PHjuGpp55SujZFydab6eq5ECKiBsyhkJg9ezaioqJgMBgwZMgQ+Pv7Y8CAAUrXpqjbGQGwJ0FEZJNDw02SJGH8+PG4efMmwsLCMHjwYIwYMULp2hRVnhHsSRAR2eZQT8LX1xcA8MADD+D06dPw9vaGh4vfYCBudyWYEUREtjnUk4iMjMSUKVPw+uuvY8KECTh79iy0Wvu75uXlYfjw4Vi7di2MRiPmzp0LjUaDNm3aYMGCBZWCxmQyYebMmbh48SI8PDwwf/58tGvXruYts6N8uEliTBAR2eRQdyA5ORljx45F27ZtkZycDFmWsWTJkvvuYzKZMGfOHHh7ewMAVqxYgcTERGzatAlGoxF79+6ttP3XX38Ns9mMzZs3IzExEe+++27NWuSg8uEmZgQRkW0Oz0k88sgjAIDY2FjExsba3SctLQ3x8fFYs2YNACAiIgL5+fkQQsBgMNzTE2nbti0sFgtkWUZhYaFDPZXasM5bK3oWIiLXpshP4szMTAQHByMmJsYaEm3atMHbb7+NVatWwd/fHz169Ki0j16vx8WLFzFo0CDcvHkT77//vkPn0mgkBAXpq12jt3fZHeRBQXoE+nja2dp9aDQeNfq8XBnbrA5qa3NdtVcSwnoxqNO8+OKLkCQJkiQhJycHbdq0walTp7B161a0b98eGzZswK+//oq5c+da91m0aBF0Oh2mTZuGy5cv46WXXsKOHTvg5eV133OZTBbk5xdVu8aNRy9g6d7fsDuxF/y9le21NCRBQfoafV6ujG1WB7W1ubbtbdLE36HtFPnpuGHDBuvXCQkJmDdvHhITE+Hn5wcACAkJwffff19pn4CAAOv6UIGBgTCbzbBYLEqUB6DCxDXHm4iIbKqzX6FTU1MxdepUaLVaeHp6Yv78+QCApKQkTJkyBWPHjkVycjJGjx4Nk8mEqVOnQq9XrivFe+mIiOxTZLipLtV0uGn9kT+wfN/v+HpSb+h1GgUqa5jU1iUH2Ga1UFub62q4ybXviKsFDjcREdmn3pC4/SczgojINvWGhGuPshER1Qn1hsTtP139CXtEREpSbUiUY0QQEdmm2pDgxDURkX3qDQlwqXAiInvUGxLWngRjgojIFoYEM4KIyCb1hgSHm4iI7FJvSHC4iYjILvWGRH0XQETkAlQdEuxEEBHdn2pDAkJwPoKIyA7VhkRZT4IxQUR0P+oNCQF4MCOIiO5LtSEhc+aaiMgu1YYEIDjcRERkh2pDQgjeSEdEZI96QwK8BJaIyB71hoQAJPYliIjuS70hAcGeBBGRHaoNCYBzEkRE9qg2JIQAU4KIyA71hgQ4J0FEZI96Q0JwToKIyB7VhgTA0SYiIntUHRJERHR/qg2JwlIzbpWY67sMIqIGTbUh8dlPV+u7BCKiBk+1IUFERPYxJIiIyCaGBBER2cSQICIimxgSRERkE0OCiIhsUjQk8vLy0LdvX5w5cwY5OTkYOXIkRo0ahbfeeguyLN+z/erVq/HCCy9g+PDh+PTTT5UsjYiIHKBYSJhMJsyZMwfe3t4AgBUrViAxMRGbNm2C0WjE3r17K21/+PBhHDt2DJs2bcL69etx5coVpUojIiIHKRYSaWlpiI+PR0hICAAgIiIC+fn5EELAYDBAq9VW2v7AgQPo0KEDEhMTMXHiRMTGxipVGhEROUhrf5Pqy8zMRHBwMGJiYrBmzRoAQJs2bfD2229j1apV8Pf3R48ePSrtc/PmTVy6dAnvv/8+Lly4gFdffRWff/45JDtLtWo0EoKC9DWutTb7uiKNxoNtVgG22f3VVXsVCYmMjAxIkoSsrCzk5ORgxowZOHXqFLZu3Yr27dtjw4YNWLx4MebOnWvdJygoCGFhYdDpdAgLC4OXlxdu3LiBxo0b3/dcFotAfn5RjWutzb6uKChIzzarANvs/mrb3iZN/B3aTpHhpg0bNiA9PR3r169HREQE0tLS0LJlS/j5+QEAQkJCcOvWrUr7dOvWDfv374cQArm5uSguLkZQUJAS5RERkYMU6UlUJTU1FVOnToVWq4Wnpyfmz58PAEhKSsKUKVPQr18/HDlyBHFxcRBCYM6cOdBoNHVVHhERVUESQoj6LqI2TCZLjbpcjy3ZBwA4Mq2Ps0tq0NTWJQfYZrVQW5tderiJiIjcA0OCiIhsYkgQEZFNDAkiIrKJIUFERDYxJIialggUAAAJaUlEQVSIyCaGBBER2cSQICIimxgSRERkE0OCiIhsYkgQEZFNDAkiIrKJIUFERDYxJIiIyCaGBBER2cSQICIimxgSRERkE0OCiIhsYkgQEZFNDAkiIrKJIUFERDYxJIiIyCaGBBER2aSt7wLqy5KhD6MUUn2XQUTUoKk2JPq0a4ygID3y84vquxQiogaLw01ERGQTQ4KIiGxiSBARkU0MCSIisokhQURENjEkiIjIJoYEERHZxJAgIiKbJCGEqO8iiIioYWJPgoiIbGJIEBGRTQwJIiKyiSFBREQ2MSSIiMgmhgQREdnEkCAiIptU+dAhWZYxb948/Pzzz9DpdEhNTUXr1q3ruyynM5lMSE5OxsWLF2E0GvHqq6/iwQcfxMyZMyFJEtq3b4+5c+fCw8O9flfIy8vD8OHDsXbtWmi1Wrdv7+rVq7F7926YTCaMGjUK3bt3d+s2m0wmzJw5ExcvXoSHhwfmz5/v1n/P2dnZ+J//+R+sX78e586dq7KdK1aswN69e6HVapGcnIzIyEjnFSBU6IsvvhAzZswQQghx7NgxMXHixHquSBlbtmwRqampQgghbty4Ifr27SsmTJggDh06JIQQIiUlRXz55Zf1WaLTGY1G8de//lUMGDBA/Prrr27f3kOHDokJEyYIi8UiCgsLxfLly92+zTt37hSTJ08WQghx4MAB8dprr7ltm9esWSOeffZZ8fzzzwshRJXtPHHihEhISBCyLIuLFy+K4cOHO7UG94jaajp69ChiYmIAAI888ghOnDhRzxUp46mnnsLrr79ufa3RaHDy5El0794dANCnTx9888039VWeItLS0hAfH4+QkBAAcPv2HjhwAB06dEBiYiImTpyI2NhYt29z27ZtYbFYIMsyCgsLodVq3bbNDzzwAN577z3r66raefToUURHR0OSJDRv3hwWiwU3btxwWg2qDInCwkL4+flZX2s0GpjN5nqsSBm+vr7w8/NDYWEhJk+ejClTpkAIAUmSrN8vKCio5yqdJzMzE8HBwdZfAAC4dXsB4ObNmzhx4gSWLVuGv/3tb3jzzTfdvs16vR4XL17EoEGDkJKSgoSEBLdt88CBA6HV3pkVqKqdd/88c3b7VTkn4efnB4PBYH0ty3Klvwh3cvnyZSQmJmL06NEYPHgw3nnnHev3DAYDAgIC6rE658rIyIAkScjKykJOTg5mzJhR6Tcqd2svAAQFBSEsLAw6nQ5hYWHw8vLClStXrN93xzZ/+OGHiI6OxrRp03D58mW89NJLMJlM1u+7Y5vLVZxnKW/n3T/PDAYD/P39nXdOpx3JhURFRWHfvn0AgB9++AEdOnSo54qUcf36dYwbNw7Tp09HXFwcAOChhx7C4cOHAQD79u3Do48+Wp8lOtWGDRuQnp6O9evXIyIiAmlpaejTp4/bthcAunXrhv3790MIgdzcXBQXF6Nnz55u3eaAgADrD8HAwECYzWa3/nddUVXtjIqKwoEDByDLMi5dugRZlhEcHOy0c6pyFdjyq5t++eUXCCGwcOFCtGvXrr7LcrrU1FT8+9//RlhYmPW9WbNmITU1FSaTCWFhYUhNTYVGo6nHKpWRkJCAefPmwcPDAykpKW7d3r///e84fPgwhBCYOnUqWrZs6dZtNhgMSE5OxrVr12AymfCXv/wFnTp1cts2X7hwAW+88QY++eQT/P7771W287333sO+ffsgyzLeeustp4akKkOCiIgco8rhJiIicgxDgoiIbGJIEBGRTQwJIiKyiSFBREQ2MSRIdWbOnIn//d//ve82mZmZmDBhQh1VVObChQvo2rVrnZ6TyB6GBBER2eSea1GQ6smyjIULFyI7OxsGgwFCCKSmpqJbt26VtnvooYfw3//939i/fz+KiorwxhtvYMCAAQCAa9euYfz48bh8+TI0Gg2WLFmCdu3a4YcffsA777wDo9GIa9euoVevXli4cGGl4/7++++Ij4/H/v37odPpYLFYEBsbiw8//BAFBQV293/vvfdw8+ZNzJkz557XBQUFWLBgAX755ReYTCb07NkTSUlJ0Gq1WL58OXbu3AlPT080atQIixYtsi52SFQT7EmQW8rOzsbVq1fx8ccf41//+heGDRuGDz744J7tLBYLfHx8kJmZiXfffRfJycnW9Z7++OMPzJo1Czt27MCjjz5qHaL65z//icmTJ+PTTz/FZ599ht27d9+zknDbtm3Rvn177N69G0DZaq0tW7ZEu3btHNr/fhYuXIiHH34YmZmZ2LZtG27evIl169bh8uXL+Oijj5CRkYHMzEz07t0bP/74Y00/QiIA7EmQm+ratSsCAwOxefNm/PHHHzh8+DB8fX2r3HbMmDEAgPDwcHTo0AFHjhwBAERGRlofRhUREYGdO3cCABYvXox9+/bh/fffx2+//YbS0lIUFRXdc9y4uDhs3boVTz31FDIzMzFy5Mj77h8UFORQ2/bu3Yvjx49jy5YtAICSkhIAQGhoKMLDwzFs2DD06dMHffr0Qc+ePR39yIiqxJAgt7R3714sWLAAL7/8Mp544gmEhYXh//7v/6rctuIaP7IsW19XXBlYkiSUr2AzZswYdOzYETExMRg0aBCys7NR1eo2gwYNwuLFi3HmzBkcOXIEixcvdnj/iucDUGmVU1mWsWzZMut6Y7du3YIkSfDw8EB6ejqOHz+OrKwsLFy4EDExMUhKSqrWZ0dUEYebyC0dPHgQ/fr1w+jRo9GpUyd89dVXsFgsVW67bds2AGUPdPn999/x2GOP2TzurVu3cPz4cbz55psYMGAArly5gvPnz0OW5Xu29fLywjPPPIOZM2diwIAB8PHxcXj/Ro0a4eTJkxBCoLCwEHv27LF+Lzo6Gh9++CGEENbH0qanp+PUqVN49tln0a5dO0yYMAFjx47F8ePHa/LxEVmxJ0FuKT4+HtOmTcPgwYNhNpvRu3dvfPnll1X+MP/+++/xySefQJZlLF26FIGBgTaPGxAQgPHjx2PYsGHQ6/UIDQ1FVFQUzp07V+XQzvPPP4/09HTMmzfP7v6tWrWy7jdkyBDs378fAwYMQGhoKLp3727tWcyaNQsLFizA4MGDYTKZ0KtXL7zyyivw9PTEoEGDMGLECOj1enh7e2P27Nm1/CRJ7bgKLKlax44dkZWV5dT194ncCYebiIjIJvYkiIjIJvYkiIjIJoYEERHZxJAgIiKbGBJERGQTQ4KIiGz6/x/oB0JkRwKIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "BernouliNB(train_bow,train_y,cv_bow,cv_y) #BoW trianing and Cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best roc auc score is 49.92769553539152\n",
      "Confusion Matrix of test set:\n",
      " [ [TN FN]\n",
      " [FP TP] ]\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEFCAYAAADpIfy5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3X98zvX+x/HHNma2mVqyJWKOJuTnskWJYzinohK2Qu34UcIWauS3MuqsWfnNTlqmZEUjlBCpQ/mx9RVpi8mwYUNnrf1mu75/rK66wlxL2zWf63nv9vnjer/fn/f1/she3nt93p/3x8FkMpkQERHDcbT1AEREpHIowIuIGJQCvIiIQSnAi4gYlAK8iIhBKcCLiBiUAryIiEEpwIuIGJQCvIiIQSnAi4gYlAK8iIhBKcCLiBhUDZt+ufOttvx6qYYe9G5v6yFINfXhiY3XdP6Fcz9Y3bZmvabX9F3VhU0DvIhIlSktsfUIqpwCvIjYB1OprUdQ5RTgRcQ+lCrAi4gYkkkzeBERgyq5aOsRVDkFeBGxD7rJKiJiUErRiIgYlG6yiogYk26yiogYlWbwIiIGVXLB1iOocgrwImIflKIRETEopWhERAxKM3gREYPSDF5ExJhMpbrJKiJiTJrBi4gYlHLwIiIGpc3GREQMSjN4ERGDUg5eRMSg9MIPERGD0gxeRMSYTCbdZBURMSbN4EVEDEqraEREDEozeBERg9IqGhERg1KKRkTEoOwwReNo6wGIiFSJ0lLrjwooLi4mOjqabt260b59ewYOHMjXX39trk9PT2fEiBF06NCBzp07ExUVxcWLlumilStXEhgYSJs2bQgODubAgQMW9db0cTkK8CJiH0yl1h8VsGjRIj744AMiIiJYu3YtTZs2Zfjw4WRmZlJcXMywYcNwcHAgPj6eiIgI1qxZw4IFC8znJyQkEBUVxdixY0lISMDHx4fhw4dz/vx5AKv6uBIFeBGxDyUXrT8qYNu2bTz00EN06dKFJk2aMHHiRPLy8khKSmLz5s1kZGQQGRmJr68vgYGBhIeHs2LFCgoLCwGIiYlh4MCB9OnTh2bNmjF79mzc3d2Jj48HsKqPK1GAFxH7UEkpGk9PTz777DPS09MpKSlh9erVODs707JlSxITE2nRogV169Y1tw8ICCA/P59Dhw5x7tw50tLSCAgIMNc7OTnh5+dHYmIiwFX7KI9usoqIfahA6iUnJ4ecnJxLyj08PPDw8LAomzZtGmPHjiUwMBAnJyccHR2ZO3cuTZo0ITMzE29vb4v29evXB+DMmTO4uLgAXLbNwYMHAa7aR3kU4EXEPlRgZh4XF8fChQsvKQ8NDSUsLMyi7MiRI7i7u7No0SK8vLxYvXo1EyZM4O2336awsBA3NzeL9s7OzgAUFRVRUFBgUfb7NsXFxQBX7aM8CvAiYh8qEOBDQkLo27fvJeV/nL2fOnWKCRMmsGzZMu6++24AWrduTWpqKvPnz8fFxcUcqH/162dXV1fzDP5ybVxdXQGu2kd5FOBFxD6YTFY3vVwq5nIOHDjAhQsXaN26tUV527Zt2b59O506dSI5OdmiLisrCyhLyzRo0MBc1rx5c4s2Xl5e5nbl9VEe3WQVEftw8aL1h5V+DbDff/+9Rfnhw4fx8fGhY8eOJCcnW+Tz9+zZg5ubGy1btsTT0xMfHx/27t1rri8pKSEpKQl/f3+Aq/ZRHgV4EbEPlbAOvk2bNvj5+TF58mR2795NWloac+fO5csvv+Tpp5+mR48eeHl5MW7cOFJSUti+fTvR0dEMGTLEnEcfOnQocXFxJCQkkJqaytSpU8nLy2PAgAEAVvVxJQ4mUwV+b/mL1XC+1VZfLdXUg97tbT0EqaY+PLHxms4vWDHJ6ra1n3zF6rY//fQTc+fOZceOHWRnZ9O8eXPGjh1rzskfP36cmTNnsm/fPjw8POjXrx9jxozB0fG3+XVsbCxxcXFkZ2fTqlUrpkyZQqtWrcz11vRxOQrwUq0owMuVXHOAj5toddvaIf++pu+qLnSTVUTsgx1uNqYALyL2QQFeRMSYTCV66baIiDHZ4QxeyySrUPe/38uXOzfw80+pHD2yhxnTnzffBXdxcSFi5gukfLeT7B8Ps2/vZgYMeMjGI5Zr4ejoyEPDH2HhtiW8l7KGhdsW80BIb4s2A0KDWPZVLO9/v4aXVkZw698aWtS713VnxKyRvPFlLKsOvU/k2jm0uaftFb/zzrtbszZtPXfe3fqKbexWJW0XXJ0pwFeRzp3uYuOGt0lOSeWhh0NYvOQtxoePYsrkMQAsWvgKI58JYf6CZfTrP4ydO/ewauUS+vfvY+ORy58VNOYxnpjwJJ+v/YzZwyLYuXEnw2c8Rd9n+gEQPPZxBjwbzLr/rGVO6Ku41nEjYtVsXOv89vj5C0sn4d8zgFWvrSTymVfISs/kxXdm0rzDHZd8n3MtZ0JfDbvq0jm7VWqy/jAIpWiqyMuzJ7N16xcMGz4OgM927OKmm26kW9fOLFkaR8iTQTz19PO8tbxsD+ht2//L35o24flxI1izZoMthy5/goODAw8Pf4S1MQmsXvg+AAd2fUNdTw8eebovn7z9MX2f7kv86++y8a2y/7+H9h5i2Zdv0SO4F+uXraNZm9tpc09bpj0+hQO7vgHgm537uc33Nh4a/jBRo1IsvnPQ+CeoWav8B1/smlI0Uhnq1fOkc+eOvPHmOxblk6e8QmDPAdSp487SmBVs/fQLi/rDh4/SpMltVTlU+Yu4erjy2Qfb+eqTLy3KM37I4IZ6N9C6cxtqu7uyd+sec13eT3l8u+dbOnTrAIDJVMqWVZtJTvzO3MZkMnE67TRejbws+r29rS//HHw/b816sxKv6jpXUmL9YRBXncEXFxfzySefkJiYyOnTpykqKsLV1RVvb2/8/f3p1asXNWroF4HytL6zBY6OjuTnFbBu7XJ6BHYhJyeXpTFxRMx6nWPHThAaZvmUnaOjI//459/5/vtUG41arkXeT3n8Z/rSS8o79vDn3Kmz1LulHgBnjlvu55154gz+Pcte/nD04FEWvWD5Wrba7rVpFdCKpB1J5rIaNWsQNmcMaxa+T8bR9L/6UoxDM3hLJ06c4MEHH2TGjBkcPXoUd3d3brnlFmrXrs2RI0eYMmUKffr0IT1df6nKU+/mmwB4K3Yu36ek0rvPEyyNiWPypDGEPz/ysue8OCOcFnfczpzoJVU5VKlEPR/rRbsu7UlY+gG167hSXFjMxQuWG1sV5OZb5OD/aMSskbjWcePDN9aZy4LCgjGVlpKw9INKG7shKAdv6aWXXsLHx4e1a9fi7u5+SX1ubi7jxo0jIiKCmJiYShvk9a5mzbI/5i1bP+eFSbMA2PH5l9x0kyeTJ40h+rWllP5udjE+fBSTJ43htdeWsvGjrTYZs/y1uj7SjZEvj2bXRzv5aPlG+o8egIlLA4mDgwOmK8w0R0Q8w98f7c5/pi/l2KEfAGjcvDGPPPMoUwZMouSicVILlcJAq2OsVe4MPikpifDw8MsGdwB3d3eef/559u3bVymDM4rc3DwANm/ZYVH+6bYvqFPHnSZNGpnL5rw6g1densLiJcuZMDGiKocpleShYQ8zdu5z7Nu2l9eenQNA/s/51HSuiVMNJ4u2Lm61yfs536KsRs0ajF80gQdCehP3ylt8tLxsTxZHR0fC5oxh66rNHP02FUcnRxydyn6kHZ0ctZrmjzSDt+Th4UFmZia+vr5XbJORkXHVt4rYu6NH0wBwdq5pUV6zZtlnk8mEg4MDb8XOY/Cgfrzy7/lMmx5Z1cOUSjB4wpMMCA1i+5ptLBg/j9KSslnkqWOncHR0xKuRF6eOnTK397rNm4yjGebPzrWcmfrWdO7s1JolkxfxyTubzHX1GtTj9ra+3N7Wl95DLJ+ZiFg1m4NfHWRqsPU7KBrdlX4zMrJyA3z//v2ZOHEiYWFh+Pv74+3tbX5XYFZWFnv37uW1114jKCioqsZ7Xfruu8Okp5+mf7/evPtugrn8gfsDycg4TVraSea8OoPBg/oRPv4l5s77jw1HK3+V3kMfYkBoEOvf/JA3X3rDoi4lKZmiwiIC/tGJtb/kzt3qunFnwJ3Ez11lbvfc/HBaBdxJdFgUuzbutOjjx8wfeb73WIuyBj638vyC8SyetJBvvzpYSVd2nTLQ6hhrlRvgw8LCcHBw4NVXXzW/HPb33NzcGDRoEGPGjKm0ARqByWRi6vR/szx2HgsXvEJCwkcEBt7Lk08MYHToJNq1bUVY2DC2bv2cr75KJMC/g/nckpISEpO+seHo5c+4sf6NhEz8F2nJx/jv+i/wbd/coj71wBE+Wr6RQeGDMZWWcupYBgNCg8nPzWdr/GYA7v5HJzrd35nta7ZxNuOsRR9FBUUcT0kj9YDlKqtf8/AZRzPI+CED+R0DpV6sVW6Ad3BwICwsjBEjRpCSkkJmZiYFBQW4uLjg7e3NHXfccdU3ikiZd95Zw4ULF5j4Qhj/Cgni5MnTjBo9kWVvrmT6tOdwdHSkZ8+u9OzZ1eK83Nw8bvC8copMqqf293XA2cWZJi18iPow+pL6wW0H8nZkHKbSUh55+lFc3FxISUpm7nOvk/9LDt6/V9lyye79A+neP9Di/OPfH+fZnqMr/0KMxA5TNHrhh1QreuGHXMm1vvAjb/pjVrd1mxl/Td9VXegJJRGxD3a4TFIBXkTsg3LwIiLGZLLDB8EU4EXEPmgGLyJiUMrBi4gYlGbwIiLGZFKAFxExKN1kFRExKM3gRUQMSgFeRMSYbLgri80owIuIfdAMXkTEoBTgRUSMyXRRDzqJiBiT/cV3BXgRsQ960ElExKjsMMA72noAIiJVorQCRwWtW7eOBx54gNatW/Pggw+yadMmc116ejojRoygQ4cOdO7cmaioKC5evGhx/sqVKwkMDKRNmzYEBwdz4MABi3pr+rgcBXgRsQumUpPVR0V8+OGHTJ48meDgYDZu3Ejv3r157rnnSEpKori4mGHDhuHg4EB8fDwRERGsWbOGBQsWmM9PSEggKiqKsWPHkpCQgI+PD8OHD+f8+fMAVvVxJQrwImIXTBdNVh9W92kyMW/ePAYPHkxISAiNGzdm5MiRdO7cmd27d7N582YyMjKIjIzE19eXwMBAwsPDWbFiBYWFhQDExMQwcOBA+vTpQ7NmzZg9ezbu7u7Ex5e9F9aaPq5EAV5E7EMlpGh++OEHMjIy6N27t0X5m2++yejRo0lMTKRFixbUrVvXXBcQEEB+fj6HDh3i3LlzpKWlERAQYK53cnLCz8+PxMREgKv2UR7dZBURu1CR933k5OSQk5NzSbmHhwceHh7mz2lpaUBZGuXpp5/m4MGDNGzYkJEjR9K9e3cyMzPx9va26KN+/foAnDlzBhcXF4DLtjl48CDAVfsojwK8iNiHCgT4uLg4Fi5ceEl5aGgoYWFh5s+5ubkATJgwgdGjRzNu3Di2bNnCqFGjePPNNyksLMTNzc2iD2dnZwCKioooKCiwKPt9m+LiYoCr9lEeBXgRsQsVmcGHhITQt2/fS8p/P3sHqFmzJgBDhgyhX79+ALRo0YJvv/2W2NhYXFxczIH6V79+dnV1Nc/gL9fG1dUV4Kp9lEcBXkTsgunqqwrN/piKuZJfUye+vr4W5bfffjvbtm2jU6dOJCcnW9RlZWWZz23QoIG5rHnz5hZtvLy8zO3K66M8uskqInbBVGr9Ya2WLVvi5uZmzpf/6vDhw9x222107NiR5ORki3z+nj17cHNzo2XLlnh6euLj48PevXvN9SUlJSQlJeHv7w9w1T7KowAvInahMgK8i4sLw4cPZ/Hixaxfv54TJ06wZMkSdu7cydChQ+nRowdeXl6MGzeOlJQUtm/fTnR0NEOGDDHn0YcOHUpcXBwJCQmkpqYydepU8vLyGDBgAIBVfVyJUjQiYh9MDpXS7ahRo3B1dWX+/PmcOXOGpk2bsmDBAjp16gTAsmXLmDlzJkFBQXh4eBAcHMzo0aPN5wcFBZGbm8u8efPIzs6mVatWxMbG4unpCUCtWrWu2seVOJhs+JqTGs632uqrpZp60Lu9rYcg1dSHJzZe0/ln7utmdVvvL3Zc03dVF5rBi4hdMJVWzgy+OlOAFxG7UFqiAC8iYkgVuXlqFArwImIXlKIRETEo2y0nsR0FeBGxC5rBi4gYlG6yiogYlGbwIiIGZaqkJ1mrMwV4EbELWiYpImJQpZrBi4gYk1I0IiIGpVU0IiIGpVU0IiIGpRy8iIhBKQcvImJQ2otGRMSglKIRETGoUt1krVq1atS05ddLNbTm6/m2HoIYlGbwIiIGpZusIiIGpRm8iIhB2eEiGgV4EbEPJaWOth5ClVOAFxG7YIe7BSvAi4h9MKEcvIiIIZXaYRJeAV5E7EKpZvAiIsakFI2IiEGVKMCLiBiTVtGIiBiUAryIiEHZYw7e/h7tEhG7VOpg/fFnHTt2jPbt27N69WpzWXp6OiNGjKBDhw507tyZqKgoLl68aHHeypUrCQwMpE2bNgQHB3PgwAGLemv6uBwFeBGxC6U4WH38GRcuXCA8PJz8/HxzWXFxMcOGDcPBwYH4+HgiIiJYs2YNCxYsMLdJSEggKiqKsWPHkpCQgI+PD8OHD+f8+fNW93ElCvAiYhdKKnD8GQsWLMDNzc2ibPPmzWRkZBAZGYmvry+BgYGEh4ezYsUKCgsLAYiJiWHgwIH06dOHZs2aMXv2bNzd3YmPj7e6jytRgBcRu1Dq4GD1UVH79u3jvffeIzIy0qI8MTGRFi1aULduXXNZQEAA+fn5HDp0iHPnzpGWlkZAQIC53snJCT8/PxITE63qozwK8CJiF0wVOCoiJyeHCRMmMHXqVG655RaLuszMTLy9vS3K6tevD8CZM2fIzMwEuGyb06dPW9VHebSKRkTsQkWWSebk5JCTk3NJuYeHBx4eHhZlL774Iu3ataNPnz6XtC8sLLwkbePs7AxAUVERBQUFFmW/b1NcXGxVH+VRgBcRu1CR1TFxcXEsXLjwkvLQ0FDCwsLMn9etW0diYiIbNmy4bD8uLi7mQP2rXz+7urri4uJiUfb7Nq6urlb1UR4FeBGxCxXZqiAkJIS+ffteUv7H2fsHH3zA+fPn6datm0X5zJkzWb58OR07diQ5OdmiLisrCyhLyzRo0MBc1rx5c4s2Xl5e5nbl9VEeBXgRsQsVmcFfLhVzOXPmzLlkJUuvXr0IDQ2ld+/e7N+/n4SEBHJycsz97dmzBzc3N1q2bImzszM+Pj7s3buXLl26AFBSUkJSUhLBwcEAdOzYsdw+yqObrCJiF0orcFjLy8uLxo0bWxwAnp6e3HrrrfTo0QMvLy/GjRtHSkoK27dvJzo6miFDhpjz6EOHDiUuLo6EhARSU1OZOnUqeXl5DBgwAMCqPq5EM3gRsQu2eN9HrVq1WLZsGTNnziQoKAgPDw+Cg4MZPXq0uU1QUBC5ubnMmzeP7OxsWrVqRWxsLJ6enlb3cSUOJpPJZu85cXNtYquvlmoq+8R2Ww9Bqqma9Zpe0/lvNhxsddth6e9c03dVF5rBi4hd0G6SIiIGVWJ/m0kqwIuIfdAMXkTEoBTgRUQMymarSWxIAV5E7MK1vMjjeqUALyJ2QSkaERGD+rMv8rieKcCLiF1QikZExKCUohERMSitohERMahSOwzxCvAiYhd0k1VExKCUgxcRMSitohERMSjl4EVEDMr+wrsCvIjYCeXgRUQMqsQO5/AK8CJiFzSDFxExKN1kFRExKPsL7+Bo6wEYVc2aNZk+43mSU3aSdfY7Pv74Xdq1a2XRZvyE0aR8v4uz55LZsOFtfH3/ZlHftGlj3nv/Dc5kfkva8SQWL4nE0/OGqrwMuQb5+QXMjl7Mfb0fx7/Hozw9bgopR36waPPxpzvo+8RIOvz9IR4IHsbK1R9a1BcXFxM5L4aufQbi3+NRxk2ZRdbZ8xZt3np3DXfec/8lx45deyr9Gq8npRU4jEIz+EoS+eo0Hn+8L9OmRXLsh+OMHPUvPt60igD/+zl5MoNJk8fw/PMjmTb13xw/kc4LL4Ty0ccr8evQk5ycn/H0vIEtW9+nsLCIMc9OIefnXF54IYxNm1Zx770PceHCBVtfolzF2Cmz+L+D3zFq6CB8/+bDR1s+I2RUOPHL5uPTuCGbPv2cF158lZDHHmXi2BHsTtzPK3OX4u7mysMP9ARgZtRCPtu5m/GhT+Hq6sLcpcsZGT6d92Pn4+TkBMDho2n4tb2T50YNtfh+n8aNqvyaqzPdZJW/hIdHHYYMeYzp0yJZ9sY7AOzatZeT6ft5fGBfFi96izFjnuLl2XNZsmQ5AF/u2ktyyi5CQoJYsOBNBg/uj5fXzdzl15Pvvz8KwN49X3Pou//yryHBvPGfd2x1eWKFQylH+HLv10wfH0bQIw8AcE+AH4NGPMfCZSuYM3MSry2O5bG+vQkPHQ5AgF87Tp3O5Mt9/8fDD/TkRPop1n+yjcgZE7i/R1cAmjdrSu/Hn2L7f3fTs9s9ABxOPca9d/vR9s4WtrnY64Q95uCVoqkEeXn5dOval7ffXm0uu3DhIiaTiVq1nPH3b0+dOu589NFWc312dg47d+6hZ8+yH+Rmtzfl5MlT5uAOcP78/zh8+Ki5jVRfx09mAGVB/ffat27Jrj1JHEo5wunMLPo/fL9FfeSLLxA5YwIAe7/+BoCu9wSY6xs3upVmPo3ZtScRgIsXSzh24iS+f/OptGsxClMFDqNQgK8EJSUlfPPNIbKzc3BwcKBx44YsXfoqJpOJ+FXraHZ72Q/jDz+csDgv7dhJmt3eFID09FPcdNONuLjUMtc7OTnRsGEDGt/WsOouRv4U7/r1ADidmWVRnnH6DLl5+SR98y1Q9nflX6PH065rHwL7PkF8wkZz27QTGdTzvBHX2i4WfTRs4E3aL/+AHDtxkuLiC+zck0TPR0Nod19vBj09jgOHUirz8q5LpZisPoxCAb6STZz0LN8l72TgoH689tpSjhz5AY86dSgsLLokj/5zbi516rgDsDbhY2rUqMGyN1+nUaNb8fK6mbnzIqhb1wNXN1dbXIpUwJ0tfGnS6FZmzVnEt8mHyfk5l/fXfcx/vyqbeeflF+Dk5EjYCy/S2d+PmNciCOzSiVnRi9j06ee/tMnHzbX2JX27utYmLy8fKEvPAJw7/yMvTRzD67On4uDoyLBnJ/LD8ZNVdLXXB91kvYyBAwfi4GDdNmwrV6685gEZzYb1m/nvF7u5r2snJk16FmfnmhQUFGIyXTpLcHBwoLS07K/XkSM/8K9/PcuCBS+T8v0uSkpKePfdBD76aCt3NG9W1ZchFeTs7Mzcl6cx4aVIHhs+BoC2d7ZgyKD+LIkt+zkpKSml/0P383TIY0BZDj79dCZL3lrJ/T26lv0dudzPnsmEg4PjL+e0ZeGrL3JPgB81a5T9OPt3aMP9wcN46901REwaVwVXe30wGWhmbq2rBviuXbsyd+5cmjZtSps2bapiTIby7bdlvyrv3LmHOu5ujB07gmlT/02tWs7UqFGDixcvmtu6u7mRk/Oz+fOG9Zv5aONWmjZtTHb2T5w79yMfb1rFj//7qcqvQyquWdPGJMQt5nTmWUpKSmjYwJvFsStxdHSk9i+pt3vvvsvinE4d2zNnYSIXLlzA3c2N/Pz8S/rNLyikjnvZb3H1bvKk2+9y9ABubq60a92C748cq6Qruz5pFc1ljBgxAnd3d6Kjo4mJiaFhQ+V/r8bL62Z69erG2rUfk5ubZy7/5ptDuLjUIjv7JxwdHWnSpBGpqb/9EDbxacSRX9ZJN2p0K3//e2dWrFhtbuPg4EDLlr6sfn991V6QVFhBYSFbd+zibr923OJ1s7n8cOoxmjVtbF7C+Mc03cWLFymbuDvSuFEDzv34PwqLinCp9du9mPRTZ+jQtuyZisT9B8k6d54HenSz6KeoqJgbb/CopKu7Phkp9WItq3LwgwYNwt/fn7lz51b2eAyhbl0PlsZE0bfvAxblgYFdyMo8y4YNWygoKKRPn17muhtu8ODeewPY8dkuABo08GLJ0iiLh6P69evNzTffxMcff1o1FyJ/Wo0aNYiIWsCmbZ+by9JPneG/X+2j2z0B3NXuTmo5O7P5s50W533x5T7uvON2atRwIsCvHSUlpezY+dsDS8dPZpB67Dh3+7UDYHfifqbMiubc+R/Nbc6d/5H/O/Add7VrXclXeX0pNZmsPozC6nXwM2fO5NChQ5U5FsM4fPgoa9d+zMuvTKGmc03Sjp3goYf/ycBB/XhmxHh+/jmXpUuWM33G85SWlnIk9RgTJoTy88+5LF/+HgD79u1n//5vWbzkVV6cEYX3LfWJiprB5s2f8dkv/whI9VWzRg0e7fNP/hMXj+eNN+Du6sprS2K58ca6PBncF3c3N556MpjFsStxd3Xlrvat+WTbFyTuP8iSOTMBuK1hA/7RvQsvRs4jNy8PjzruzF26HN+/+dD9vk4ABD38APEJGxk1fgYjhwyi+MIFlsSu5Ia6dRjU/yFb/hFUO8YJ29ZzMF3ubl8VcXNtYquvrnS1a7swefJY+vXvjbf3zaSkpPJq5ELWrdsElC15nDEjnMFP9MfNzZU9u5MID3+Jw4d/W/fesGEDol97iS5dAsjLy+eDNRt56aU5FBQU2uqyKl32ie22HsJfprCoiLlL3uKT7V9QXHwB/w5teX70MBrdeou5zTurP+TdNes5k3WWxo1uZfSwwfToeo+5Pr+gkFfnx7Dls52YTCbuvqsdk8aOpP7NN5nbHD12nOjFsew/+B2lpaV09u/A+NCnuMW7fpVeb2WrWa/pNZ0/sHFfq9u+e3ztNX1XdaEAL9WKkQK8/LWuNcA/3vgRq9uuOr7O6ra5ubnMnz+fTz/9lP/973/4+PgwevRoAgMDAUhPTyciIoJ9+/bh4uJC3759GTduHDVq/JZAWblyJbGxsZw9e5YWLVowZcoUi0Ut1vRxOVoHLyJ24SImq4+KmDRpEjt27GDWrFmsW7eOXr16ERoayldffUVxcTHDhg3DwcGB+Ph4IiIiWLNmDQsWLDCfn5CQQFRUFGPHjiUhIQEfHx+GDx/O+fNlm8pZ08eVKMCiVQpVAAAF7klEQVSLiF0wVeA/a509e5YtW7YwefJkOnfuTOPGjXnmmWfw9/dnzZo1bN68mYyMDCIjI/H19SUwMJDw8HBWrFhBYWFZqjUmJoaBAwfSp08fmjVrxuzZs3F3dyc+Ph7Aqj6uRAFeROxCZTzJWrt2bd544w3uusvyeQYHBwd++uknEhMTadGiBXXr1jXXBQQEkJ+fz6FDhzh37hxpaWkEBPz2LIOTkxN+fn4kJpY99Xy1Psqj3SRFxC5U5HZjTk4OOTk5l5R7eHjg4fHb8wXu7u7cd999Fm3279/P7t27mTp1Kjt37sTb29uivn79spvfZ86cwcWlbJ+hy7U5ePAgAJmZmeX2UR4FeBGxCxXZRCwuLo6FCxdeUh4aGkpYWNgVzzt69CihoaG0bduW4OBgPv30U9zc3CzaODs7A1BUVERBQYFF2e/bFBcXA1BYWFhuH+VRgBcRu1CRrQpCQkLo2/fSZZW/n73/0b59+wgNDaVBgwbExMRQs2ZNXFxczIH6V79+dnV1Nc/gL9fG1bVsO4qr9VEeBXgRsQsVmcH/MRVzNevXr2fy5Mn4+/szf/583N3LdoX19vYmOTnZom1WVpa5rkGDBuay5s2bW7Tx8vKyqo/y6CariNgFk8lk9VERGzZsYMKECdx///3ExMSYgztAx44dSU5Otsjn79mzBzc3N1q2bImnpyc+Pj7s3bvXXF9SUkJSUhL+/v5W9VEeBXgRsQuVsYrmzJkzTJs2jYCAAMaPH092djZnz57l7NmzZGdn06NHD7y8vBg3bhwpKSls376d6OhohgwZYs6jDx06lLi4OBISEkhNTWXq1Knk5eUxYMAAAKv6uBKlaETELlTGfvBbtmyhoKCA3bt306VLF4u6Dh06sGrVKpYtW8bMmTMJCgrCw8OD4OBgRo8ebW4XFBREbm4u8+bNIzs7m1atWhEbG4unpycAtWrVumofV6KtCqRa0VYFciXXulVBj0b/sLrtpyc3X9N3VReawYuIXSgx2d+O8ArwImIX9Mo+ERGDMtKLPKylAC8idsH+wrsCvIjYiYo86GQUCvAiYhcU4EVEDEqraEREDEqraEREDMqGz3TajAK8iNgF5eBFRAxKM3gREYMqqdA+kcagAC8idkFPsoqIGJRW0YiIGJRm8CIiBqUZvIiIQWkGLyJiUNqqQETEoJSiERExKJNm8CIixqStCkREDEpbFYiIGJRm8CIiBlVSqhy8iIghaRWNiIhBKQcvImJQysGLiBiUZvAiIgalm6wiIgalFI2IiEEpRSMiYlDaLlhExKC0Dl5ExKA0gxcRMahSbRcsImJMuskqImJQ9hjgHUz2eNUiInbA0dYDEBGRyqEALyJiUArwIiIGpQAvImJQCvAiIgalAC8iYlAK8CIiBqUALyJiUArwIiIGpQBvQ6WlpcyfP58uXbrQtm1bhg4dyvHjx209LKlGYmJiePzxx209DLlOKcDb0KJFi1i1ahWzZs3ivffew8nJiWHDhlFUVGTroUk1sHLlSl5//XVbD0OuYwrwNlJcXExsbCyhoaF07dqVO+64g9dff51z586xadMmWw9PbCgzM5NnnnmGOXPm4OPjY+vhyHVMAd5GkpOTyc/P5+677zaXubu707JlSxITE204MrG1Q4cO4ebmxvr162nbtq2thyPXMW0XbCOZmZkAeHl5WZTXr1+f06dP22JIUk10796d7t2723oYYgCawdtIQUEBAM7Ozhblzs7OFBcX22JIImIwCvA24uLiAnBJMC8uLsbV1dUWQxIRg1GAt5FbbrkFgKysLIvyrKysS9I2IiJ/hgK8jdxxxx24u7uzd+9ec1lubi7fffcd/v7+NhyZiBiFbrLaiLOzM4MHD+b111+nXr16NGzYkOjoaLy8vOjVq5ethyciBqAAb0PPPvssJSUlTJ8+nYKCAvz8/Fi2bNklN15FRP4MvXRbRMSglIMXETEoBXgREYNSgBcRMSgFeBERg1KAFxExKAV4ERGDUoAXETEoBXgREYNSgBcRMaj/B06cb6/MWc1KAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "best_alpha=1\n",
    "testBernouli(train_bow,train_y,test_bow,test_y,best_alpha) #BoW testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tPositive\t\t\t\t\tNegative\n",
      "________________________________________________________________________________________________\n",
      "\t-19.6946\tabhor          \t\t\t\t-1.1539\tlitmus         \n",
      "\t-19.6946\tabod           \t\t\t\t-1.1600\tteeco          \n",
      "\t-19.6946\taborb          \t\t\t\t-1.3328\tgrade          \n",
      "\t-19.6946\taboveour       \t\t\t\t-1.3668\tlustrous       \n",
      "\t-19.6946\tabras          \t\t\t\t-1.3970\tgrown          \n",
      "\t-19.6946\tabsenc         \t\t\t\t-1.4450\tflower         \n",
      "\t-19.6946\tabsoluet       \t\t\t\t-1.4484\tordor          \n",
      "\t-19.6946\tabysm          \t\t\t\t-1.4966\tprotool        \n",
      "\t-19.6946\tacademia       \t\t\t\t-1.5091\tvegetariani    \n",
      "\t-19.6946\tacana          \t\t\t\t-1.5223\ttryptophan     \n",
      "\t-19.6946\taccept         \t\t\t\t-1.6964\tmanipularlo    \n",
      "\t-19.6946\taccostom       \t\t\t\t-1.7050\tgiraff         \n",
      "\t-19.6946\tachiev         \t\t\t\t-1.8514\tyen            \n",
      "\t-19.6946\tacia           \t\t\t\t-1.8587\tbyt            \n",
      "\t-19.6946\tactor          \t\t\t\t-1.9351\ttokyo          \n",
      "\t-19.6946\tacupunctur     \t\t\t\t-1.9589\tamac           \n",
      "\t-19.6946\tadd            \t\t\t\t-1.9720\trecom          \n",
      "\t-19.6946\taddison        \t\t\t\t-2.0287\tflaki          \n",
      "\t-19.6946\taddit          \t\t\t\t-2.0306\tmusket         \n",
      "\t-19.6946\tadio           \t\t\t\t-2.0422\tdrambui        \n"
     ]
    }
   ],
   "source": [
    "def show_most_informative_features(vectorizer, clf, n=20):\n",
    "    feature_names = uni_gram.get_feature_names()\n",
    "    \n",
    "    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))\n",
    "    top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])\n",
    "    print(\"\\t\\t\\tPositive\\t\\t\\t\\t\\tNegative\")\n",
    "    print(\"________________________________________________________________________________________________\")\n",
    "    for (coef_1, fn_1), (coef_2, fn_2) in top:\n",
    "        print(\"\\t%.4f\\t%-15s\\t\\t\\t\\t%.4f\\t%-15s\" % (coef_1, fn_1, coef_2, fn_2))\n",
    "        \n",
    "show_most_informative_features(train_bow,bnb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF vectorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading tf-idf data\n",
    "\n",
    "train_tfidf=sparse.load_npz(\"train_tfidf.npz\")\n",
    "cv_tfidf=sparse.load_npz(\"cv_tfidf.npz\")\n",
    "test_tfidf=sparse.load_npz(\"test_tfidf.npz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Best alpha Value 0.05 with highest roc_auc Score is 73.83571835772688\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEhCAYAAACOZ4wDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XlclNX+B/DPDDPMsIwsNqJdzC0HZFFwxy1cwBSt3K2rXreruVzvVQvL0vzd1OwaWlqWJphpWWqGmpqZS25p4o6pmIpbCIQYIMvAzPn9AUyMIDMIw8DM5/168ao5z/PMfM+o8+E855nnSIQQAkREROWQWrsAIiKq+RgWRERkEsOCiIhMYlgQEZFJDAsiIjKJYUFERCYxLIiIyCSGBRERmcSwoBplzpw58PHxwfvvv2/tUmqtf//731iyZIm1yyAbw7CgGiMnJwc7d+6ERqPB5s2bkZ+fb+2Sap2CggIcPXoU3bp1s3YpZGMYFlRjfP/998jJycG8efOQmpqKH3/80dol1TqnT5+GRCJBcHBwpZ5n9+7dGDRoEFq2bImwsDD88MMPAICRI0di3rx5VVAp1TYMC6oxNm/ejDZt2qBNmzbw9/fHhg0bSu0jhMC6desQERGBli1bomfPnvjoo4+g0+nM2g4APXr0wCuvvGL0vEePHoWPjw+OHz9u2GfRokUYN24cWrVqhf/85z/Izc1FVFQUwsPDERAQgNatW2PMmDH49ddfza7x3XffRWBgIP7880+jY7788ksEBATg3r17pfo8d+5cdOzYEQUFBUbtH374IVq1aoWsrCxD24EDB9ClSxc4ODjg4sWLGDt2LNq1a4egoCC89NJLOHLkiMk/h+XLl2PatGlo3LgxZs2aBWdnZ7z++uv45ZdfcOrUKYwfP97kc5DtYVhQjZCYmIi4uDg8//zzAICBAwfi+PHjuHr1qtF+S5cuxYIFCxASEoLly5dj2LBh+Pjjj7F06VKztlfE+vXr4e3tjeXLl2P48OGYNWsWNm7ciLFjx+LTTz9FZGQkEhISMH36dJS8H2d5NQwZMgRarRY7duwweq1vv/0W3bt3h6enZ6k6nn/+eaSnp5f6oN+xYwd69uwJV1dXQ9vBgwfRrVs3ZGVlYcyYMVAoFFi8eDHef/99yOVyTJw4Ebdu3Xpkn0+dOoUPP/wQI0eORFRUFP7+97/j9ddfR1ZWFt5++23069cP3t7eFX4vyQYIohrgvffeE0FBQSIrK0sIIcT9+/dFQECAmD9/vmGfjIwM4e/vL9566y2jY5csWSKGDh1qcrtOpxNCCNG9e3cxc+ZMo32OHDkiNBqNOHbsmGGfLl26iPz8fCGEEHl5eWL06NEiNjbW6Ljo6Gih0WhEUlKSWTXqdDrx4osviqFDhxq2XblyRWg0GrF///5Hvj+9evUSr776quFxfHy80Gg04sCBA4a233//Xfj6+oq0tDRx5swZodFoxMmTJw3b09LSxMKFC0VCQsIjX2fatGkiODhYZGRkGNpu3rwpNBqN8PX1Fb/99tsjjyXbJrN2WBHpdDp8++236NGjBwDgwYMHkMlkeOaZZxAbG4sZM2bAyckJZ86cQX5+Pnr37m10/PTp0wEAhw4dKnd7RTVr1gwyWeE/EUdHR6xZswYA8McffyAxMRHXr1/HgQMHAABarRYATNYIAIMHD8brr7+O69evo0mTJvjmm2+gVqvRtWvXR9by/PPPIyYmBrm5uVAqldi+fTvUajW6dOli2OfAgQMIDAyEp6cnlEol1Go1Jk2ahL59+6JLly7o0KEDXn/99Ue+hk6nw+HDh9G1a1eoVCpDu0QiAQCEhYWhWbNm5rx1ZIN4Goqs7uDBg0hNTcV3332H1q1bG3727NmDjIwMwymb9PR0AMATTzxR5vOY2l5RDz/P0aNH0b9/f3Tu3BkTJ05EbGwsHB0dAcBwGsqcGvr06QNXV1fExsaioKAA27dvxwsvvAAHB4dHHvP8888jOzsb+/fvh16vx86dO9GvXz+jY3766SfDVVDOzs7YsGEDwsLCsHv3bkyePBkhISGYOXMmMjMzy3yNmzdvIisrC/7+/kbtxX0aMWLEI+sj28eRBVnd5s2b0aBBA7z33nultkVGRmLDhg0YPHgw6tSpAwClJoFTUlJw9epVk9uDg4OhVCoBAHq93mifkpPEZbl58yZefvll9OrVCytWrIC3tzckEgm++OILHDp0yLCfOTU4OTkhIiICu3btQseOHZGamoqBAweW+/oNGzZEmzZtsGvXLnh4eCA5OdkwvwMUjmyOHz+OqVOnGh0zf/58CCFw6dIl7Ny5E9HR0XBzc8PcuXNLvcb9+/cBAG5ubkbtK1euBAB4eHiUWyPZNo4syKrS0tLw008/oW/fvmjbtm2pn+eeew7x8fE4f/48WrZsCblcjj179hg9xxdffIFJkyZBo9GUu734t39XV1f8/vvvRvv88ssv5dYZHx+PvLw8jB07Fg0bNjScmjl48CCAv8LHVI3FNQwePBg3btzAihUrEBwcjKZNm5p8r1544QUcOnQIW7duhUajQYsWLQzbjh8/DmdnZ8Oo4Pvvv0fHjh2RkpICiUSCFi1aYObMmWjatCnu3LlT5vMXB11CQoKh7aeffjL0JTc312SNZLsc5vGiabKiDRs24NChQ5g7dy7UanWp7Q0aNMC6deug0+nQr18/ZGVl4fPPP8eDBw8ghMDu3bvxySef4J///Cd69epV7vbOnTsDAJKTk7F9+3YIIaDT6bBp0yZs374d2dnZGDBgALy9vbF27Vo0aNAA4eHhAArnLDZs2ICbN2/Cw8MDv/32G9577z3DnMWgQYOgVqvh5ORkVg1eXl7Ys2cPzp07h6lTp5Y69VOWRo0aITo6Gr/++ivGjh2L1q1bG7atW7cOTz75JHr16gWg8IP/yy+/xMGDB6FSqZCSkoIvv/wSe/fuxaRJk+Dr61vq+T08PPDdd9/h6NGj0Gq1OHPmDBYsWICePXvit99+Q3p6Ory9veHl5VWxP2SyDdabWycSok+fPqJ3797l7jN8+HDRsmVLcf/+faHX60VMTIwICwsT/v7+Ijw8XKxZs0bo9XohhDC5XQghsrOzxVtvvSXat28vgoKCxMsvvyxOnjxZ6mqoh6+Y2rVrl4iIiBCBgYGiS5cuYsqUKeKXX34RPj4+Yu3atYb9zKlBiMIrpIKCgkRmZqbZ79f06dNFixYtRHJyslF7WFiY2Llzp1FbfHy8GD9+vOjQoYMICAgQ/fv3F5s3by73+S9evCiGDRsmAgICRJs2bcSCBQtEQUGBWLhwoQgICBAff/yx2bWSbZEIUeICcSKqNgMHDoSfnx/mz59v7VKITOIEN1E1evDgAdasWYMLFy7g8uXLWLx4sbVLIjILw4KoGimVSmzcuBFarRb/93//x+8tUK3B01BERGQSL50lIiKTGBZERGQSw4KIiEyyiQnu9PQH0OsrPvVSt64r0tLKv82DrWGf7QP7bPsq01+pVAIPD5cKHWMTYaHXi8cKi+Jj7Q37bB/YZ9tXnf3laSgiIjKJYUFERCYxLIiIyCSGBRERmcSwICIikxgWRERkkt2GxY6fEzH49e+sXQYRUa1gt2HxzU/XkKfVITNba+1SiIhqPLsNC2dF4fcRs3LyrVwJEVHNZ7dhoVQ4AAAe5BRYuRIioprPbsNCIS8Mi6xcjiyIiEyx27BwKjoN9YCnoYiITLLbsFA6Fp2GyuVpKCIiU+w2LIpPQ2XzNBQRkUl2GxbFOMFNRGSaxdazOH78OEaNGlXmNm9vb+zduxdRUVFYtWpVqe0XLlyATFY9S2084MiCiMgki30iBwcH4/Dhw0ZtCQkJmDBhAiZOnAgAuHz5MoYOHYpp06YZF1UNQSGK1gzh1VBERKZZ7FPZ0dERarXa8Dg/Px8LFy5EWFgYhg4dCqAwPLp37260X3XjaSgiItOqbVnVdevWISkpCTExMQCAjIwMJCUl4emnn66uEsrE01BERKZVywR3Tk4OVq5ciVGjRsHLywtA4agCALZv347w8HB0794ds2bNQkpKSnWUZJDNS2eJiEyqlpHF1q1bkZeXZzThXRwWrq6uWLZsGVJTU7F06VKMHDkSsbGxcHJyMvv569Z1rXBNcse/Lp2tW9cVUqmkws9RW6nVKmuXUO3YZ/tgb32uzv5WW1iEhYXB09PT0Pbiiy8iIiICbm5uAABfX19oNBo888wz+PHHH9G/f3+znz8tLQt6vahQTdq8whGFXgC37qTDWSmv0PG1lVqtQmpqprXLqFbss32wtz5Xpr9SqaTCv2Rb/DTUvXv3cObMGfTr18+oXSKRGIKimJeXF9zd3ZGUlGTpsoxk8VQUEVG5LB4Wp06dgkQiQbt27Yza58+fjxdeeMGo7datW0hPT6+WSe+S4xDeH4qIqHwWD4tff/0VDRs2hLOzs1H7s88+iytXrmD+/PlITEzEL7/8gqlTp6Jly5YIDQ21dFlGeEUUEVH5LB4WqamppU43AUDbtm3xySef4Pz58xgwYAD+9a9/wd/fH6tWrYJUWj13IXGUFb4Or4giIiqfxSe433777Udu69q1K7p27WrpEh7J2UkObWYecrU6q9VARFQb2PWNBF2UhVmZk8eRBRFReew6LJQKhgURkTnsOiwkEgmUjg7IyeNpKCKi8th1WACFy6vmaDmyICIqD8NCIUMuT0MREZWLYeHowDkLIiIT7D4slAoZcnjpLBFRuew+LDiyICIyrdoWP6pplI4OgERSOGfBkQURUbnsNixe6qWBu4czvtx1EdkcWRARlctuT0PVcXFEXTcnKB0dkKfVVXg9DCIie2K3YVHMqehb3DwVRUT0aAwLQ1jwVBQR0aMwLIrCgvMWRESPxrBwdAAA5PL+UEREj2T3YWG48yxPQxERPZLdh4UTb1NORGQSw6LoNBTDgojo0RgWhpEF5yyIiB7F7sNCUTzBzTkLIqJHsvuwkHK1PCIik+w+LICi1fI4Z0FE9EgMC3BpVSIiUyx219njx49j1KhRZW7z9vbG3r17cfv2bbz99ts4ceIElEolBgwYgOnTp0Mmq96b4To5OnBpVSKicljsUzk4OBiHDx82aktISMCECRMwceJEaLVajBs3Dk2aNMFXX32FW7duYfbs2ZDJZJg+fbqlyiqTkqehiIjKZbHTUI6OjlCr1YYfd3d3LFy4EGFhYRg6dCh2796NO3fu4N1334VGo0HPnj3xyiuv4PPPP0dubq6lyiqTs0KGB7kMCyKiR6m2OYt169YhKSkJr7/+OgAgLi4OLVq0gJubm2GfDh06IDs7GxcuXKiusgAAbq6OuJ+ZByG4pgURUVmqJSxycnKwcuVKjBo1Cl5eXgCA5ORk1K9f32i/evXqAQDu3r1bHWUZeKqUyMvX8fJZIqJHqJaw2Lp1K/Ly8owmvHNzc+Ho6Gi0X/HjvLy86ijLwF1V+LrpWdX7ukREtUW1XHa0detWhIWFwdPT09CmVCqh1WqN9it+7OzsXKHnr1vX9bFrU6tVaNqw8HX1UinUatVjP1dtYQ99fBj7bB/src/V2V+Lh8W9e/dw5swZvPzyy0bt9evXx8WLF43aUlJSDNsqIi0t67HW0FarVUhNzQR0haefEm+no6GnU4WfpzYx9NmOsM/2wd76XJn+SqWSCv+SbfHTUKdOnYJEIkG7du2M2tu1a4eLFy8iIyPD0Hb8+HG4uLjAz8/P0mUZ8XAtPA11P5OnoYiIymLxsPj111/RsGHDUqeWevXqBS8vL0yfPh2XLl3Cvn37EBUVhTFjxpSay7A0ucwBrk5ypDMsiIjKZPGwSE1NNbo8tphCocDq1asBAEOHDsXcuXMxbNgwTJkyxdIllclTpWBYEBE9gsXnLN5+++1HbmvUqBGio6MtXYJZ3BkWRESPxBsJFvFQKXjpLBHRIzAsinioFMjMzkd+gd7apRAR1TgMiyIergoAwH2OLoiISmFYFPGoUxgWnLcgIiqNYVGkeGTBsCAiKo1hUcRDpQTAsCAiKgvDooiTwgEKuQPDgoioDAyLIhKJhJfPEhE9AsOiBA+VAumZ1btKHxFRbcCwKMFDpeDNBImIysCwKMFDpcD9LC30XF6ViMgIw6IED5UCOr1A5gOt6Z2JiOwIw6KE4u9a3OOpKCIiIwyLEtxVRbf8YFgQERlhWJTgqeLIgoioLAyLElQujnCQSngzQSKihzAsSpBKJHB3dcS9DIYFEVFJDIuHuKsUHFkQET2EYfEQD5WScxZERA9hWDzEw7XwW9yCX8wjIjJgWDzEQ6VAXr4OOXk6a5dCRFRjMCwe4qEqXgSJNxQkIipmVlj861//wtGjRy1dS41gCAtOchMRGZgVFmFhYVixYgV69+6N6Oho3L9/39J1WY0hLHj5LBGRgVlh8dxzz2H9+vVYsWIF0tLSMHjwYLz66qs4d+6cyWNjY2PRt29fBAYGIiIiArt27TJsi4qKgo+PT6mfgoKCx+9RJbm7cmRBRPQwmbk76vV63LhxA4mJidDpdKhbty7mzZuH0NBQTJs2rcxjtm7ditmzZ2PWrFkIDQ3Fzp07MWPGDNSrVw9t2rTB5cuXMXTo0FLHy2Rml1Xl5DIpVM5yLq9KRFSCWZ/KS5cuxZYtW9CwYUO89NJL+OCDDyCXy5GdnY3u3buXGRZCCHzwwQcYMWIE/vGPfwAAJk2ahLi4OBw7dgxt2rRBQkICunfvDrVaXbW9qiQPVwXDgoioBLPC4t69e/j000/h6+tr1O7s7IyoqKgyj7l27Rru3LmDfv36GbVHR0cDADIyMpCUlISnn376ceq2KA+Vgl/MIyIqwaw5iylTpuCrr74CUBgCkydPRmpqKgCgS5cuZR6TmJgIANBqtZgwYQJCQkIwZMgQ7Nu3DwCQkJAAANi+fTvCw8PRvXt3zJo1CykpKZXqUFUoXIubYUFEVMyskcVrr72GHj16AAD+9re/oX379pg9ezY+/fTTRx6TlZUFAIiMjMSUKVMwffp0/PDDD5g8eTKio6Nx48YNAICrqyuWLVuG1NRULF26FCNHjkRsbCycnJzM7kTduq5m7/swtVpVqs27fh0cOPM73Nyd4Sh3eOznrqnK6rOtY5/tg731uTr7a1ZYpKenY9SoUQAAhUKB0aNHIzY2ttxj5HI5AGDMmDEYNGgQAKBFixaIj49HTEwMVq9ejYiICLi5uQEAfH19odFo8Mwzz+DHH39E//79ze5EWloW9PqK355DrVYhNTWzdO1F460riWmo525+aNUGj+qzLWOf7YO99bky/ZVKJRX+Jdus01A6nQ7JycmGx3/88YfJeyfVr18fAKDRaIzamzdvjtu3b0MikRiCopiXlxfc3d2RlJRkVvGW4qlSAuCKeURExcwaWYwePRovvPACunbtColEgqNHjyIyMrLcY/z8/ODi4oLz58+jQ4cOhvaEhAQ89dRTmD9/PuLi4oxGKLdu3UJ6errVJ73dDSvm8ZYfRESAmWExePBgBAQE4NixY3BwcMC4ceNKjRgeplQqMX78eKxYsQL16tVDUFAQduzYgcOHD2PNmjWQy+XYsGED5s+fjxEjRiAlJQULFixAy5YtERoaWhV9e2yehrW4tVatg4iopjD722/169dH7969IYSATqfDkSNH0Llz53KPmTx5MpydnbFs2TLcvXsXTZs2xfLlyxESEgIA+OSTT/Dhhx9iwIABcHR0RM+ePfHqq69CKrXu/Q2dFDIoHB04siAiKmJWWHzwwQdYtWpV4QEyGbRaLZ5++mls377d5LGjR4/G6NGjy9zWtWtXdO3a1fxqq5GnSsE5CyKiImb9Cr9161bs378fvXv3xu7du/HOO+9YfV7B0tz5LW4iIgOzwsLT0xP16tVD06ZNcenSJbzwwguGL9XZKg+VgjcTJCIqYlZYyGQy3Lx5E02bNkVcXBwKCgqQl2fbH6QeKgXuZ2of6/sbRES2xqywePnllzFnzhyEhoZiz549CA0NRceOHS1dm1V5qBTQC4GMbF4RRURk1gR3QUEB1q5dC6BwfYobN27Ax8fHooVZ21/Lq+YZ1rggIrJXZo0sli5davh/Jycn+Pr6QiKRWKyomqBkWBAR2TuzRhYajQYff/wx2rZtC2dnZ0O7v7+/xQqzNo+iW34wLIiIzAyLs2fP4uzZs9i0aZOhTSKRYO/evRYrzNpUznI4SCUMCyIimBkWxWtQ2BOpRMLvWhARFTErLNasWVNm+5gxY6q0mJqmcBEk3vKDiMissCj5BTytVosTJ04Y7u9kyzxUCtxMybJ2GUREVmdWWLzzzjtGj5OTk/HGG29YpKCaxEOlwNmrhWt32PrVX0RE5Xms27t6eXnhzp07VV1LjeOhUkCbr0dOXoG1SyEisqoKz1kIIRAfH4+6detarKiawsOwCFIenJVyK1dDRGQ9FZ6zAIAGDRqYXCnPFhR/c/t+Zh681RVbr5aIyJaYPWdx4sQJtGvXDvfv30dcXJxhjW1b5lliZEFEZM/Mvt3HsmXLAAC5ublYtWoVVqxYYdHCagJ31V8jCyIie2ZWWOzduxcxMTEACpdXXb9+PXbu3GnRwmoCmYMUdZzlHFkQkd0zKyzy8/Mhl/81wSuXy+3mUlJ3lQL3uQgSEdk5s+YsWrdujZkzZ2Lw4MGQSCSIjY1Fq1atLF1bjeCpUiItg9/iJiL7ZtbIYs6cOVCr1XjnnXfwv//9D0888YRdfCkPKBxZ8P5QRGTvzBpZODs7o2fPnnjttdcMV0M5OTlZurYawUOlQFZOPvILdJDLHKxdDhGRVfBqKBM8XLkIEhGRxa+Gio2NRd++fREYGIiIiAjs2rXLsO327duYOHEiWrdujU6dOmHx4sUoKKhZt9bwqMOwICKy6NVQW7duxezZszFs2DB899136NevH2bMmIGTJ09Cq9Vi3LhxkEgk+Oqrr/D2229j8+bNWL58+eP3xgI4siAiesyrob799luTV0MJIfDBBx9gxIgR+Mc//gEAmDRpEuLi4nDs2DH8/vvvuHPnDjZu3Ag3NzdoNBq88sorWLhwISZNmgSlUln53lUBw1rcvHyWiOxYha6GWrRoEf73v/9BrVbjzTffLPeYa9eu4c6dO+jXr59Re3R0NKZMmYK4uDi0aNECbm5uhm0dOnRAdnY2Lly48BhdsQwnhQxKRwekZzAsiMh+mRUWly9fRmJiItzc3ODi4oLTp0/j2WefLfeYxMREAIWLJU2YMAEhISEYMmSIYYnW5OTkUveXqlevHgDg7t27Fe2HRXmoFBxZEJFdM+s01Jtvvonnn38eu3fvxvDhw7F3716Eh4eXe0xWVuEKc5GRkZgyZQqmT5+OH374AZMnT0Z0dDRyc3Ph4uJidIyjoyMAIC+vYh/Mdes+/h1h1WqVyX28PF2QlVNg1r61ga30oyLYZ/tgb32uzv6aFRYSiQQTJkxAeno6mjZtiv79+2PQoEHlHlM8IT5mzBjDvi1atEB8fDxiYmKgVCqh1WqNjil+7OzsXKFOpKVlQa8XFToGKHyjU1MzTe7nonDAjbsZZu1b05nbZ1vCPtsHe+tzZforlUoq/Eu2WaehikcATz31FK5cuQKlUgmptPxDi08xaTQao/bmzZvj9u3bqF+/PlJSUoy2FT+uabc/96ijwJ9Z2scKJCIiW2BWWLRs2RL/+c9/0LFjR8TExGDRokWQycoflPj5+cHFxQXnz583ak9ISMBTTz2Fdu3a4eLFi8jIyDBsO378OFxcXODn5/cYXbEcD1cF9ELgzwda0zsTEdkgs8Ji9uzZGD16NJo0aYLZs2dDr9cjKiqq3GOUSiXGjx+PFStWYNu2bbh58yY+/vhjHD58GGPHjkWvXr3g5eWF6dOn49KlS9i3bx+ioqIwZswYw9xFTeGhKryMl3efJSJ7ZfacRVBQEAAgNDQUoaGhZj355MmT4ezsjGXLluHu3bto2rQpli9fjpCQEADA6tWr8d///hdDhw5FnTp1MGzYMEyZMuXxemJBhrW4M/LQpIGViyEisgKzwqIyRo8ejdGjR5e5rVGjRoiOjrZ0CZVWHBYcWRCRvTLrNJS9c3WWw0Eqwb1MrmtBRPaJYWEGqUQCD5WCa3ETkd1iWJiJiyARkT1jWJjJk2FBRHaMYWEmd9fCsBCCX8wjIvvDsDCTh0oBbYEe2Xk1a3EmIqLqwLAwk2FdC96qnIjsEMPCTFwEiYjsGcPCTIaw4CQ3EdkhhoWZ3LkWNxHZMYaFmWQOUtRxcWRYEJFdYlhUgIcrv2tBRPaJYVEBHvxiHhHZKYZFBRSGBW8mSET2h2FRAR4qBR7kFkCbr7N2KURE1YphUQH8rgUR2SuGRQUYFkHivAUR2RmGRQUYlldlWBCRnWFYVEDxF/M4siAie8OwqAAnhQxOCgeOLIjI7jAsKsjdlcurEpH9YVhUkKdKwauhiMjuMCwqiGtxE5E9YlhUkIdKiT+ztNDp9dYuhYio2sgs+eTXrl1Dnz59SrXPnz8fQ4YMwYwZM7Bjxw6jbV5eXjh48KAly6oUD5UCeiGQ8SDfcCktEZGts2hYXL58Ga6urvj++++N2lUqlWH7tGnTMHToUMM2BwcHS5ZUaSUXQWJYEJG9sGhYJCQkoFmzZlCr1aW2abVaJCYmIjAwsMztNZWHYRGkXAB1rFsMEVE1seicxeXLl9GsWbMyt129ehUFBQV4+umnLVlClfOowxXziMj+WDQsEhISkJKSguHDh6NTp0546aWXcPjwYQCFQSKTybBy5Ur06NEDYWFhmD9/PjIzMy1ZUqWpnOSQOUgYFkRkVyx2Gio7Oxu3b9+Gp6cnZs6cCRcXF2zbtg3jx49HTEwMrly5AgDw9vbGJ598ghs3buDdd9/FpUuX8Pnnn0MqNT/H6tZ1few61WpVhY+p6+aEnHz9Yx1bE9TWuiuDfbYP9tbn6uyvRAghLPXkDx48gFwuh6Ojo6Ft3LhxEEJg9erVyMrKQp06f533P336NIYPH44NGzagdevWZr9OWloW9PqKd0OtViE1teIjmXfWn4RUIsGsv5tfY03xuH2uzdhn+2Bvfa5Mf6VSSYV/ybboaSgXFxejoAAAjUaD33//HVKp1CgoAMDHxwcAZ6F2AAAUHklEQVQAkJSUZMmyKs2D3+ImIjtjsbA4ffo0goODce7cOaP2+Ph4NG/eHFOmTMGkSZOMthXvW9MnvYvX4n6c0QwRUW1ksbAICAiAt7c35syZg5MnT+Lq1auYP38+Tp8+jUmTJqFv377Yt28fVq1ahZs3b+LAgQOYPXs2evfubRhh1FQab3fkF+hx/GKytUshIqoWFpvglsvlWL16NaKiojBt2jRkZGTA398fMTEx8PPzg5+fH/R6PVavXo2PPvoIKpUKERERmD59uqVKqjKtmj8Bb7ULth9JRIcWXpBKJdYuiYjIoiw6wV1dqnuCGwDiLqVgRWw8JvT3Q0f/+o/1HNZgb5OAAPtsL+ytzzY1wW3LWvuo8Te1C7YfTeTcBRHZPIbFY5JKJHiucxMkpWXjxKUUa5dDRGRRDItKaOOjxt+ecMG2I9c5uiAim8awqASpRIL+nRsjKS0bcZc5uiAi28WwqKS2vvXw5BMu2HYkEfraf60AEVGZGBaVJJVI0L9TY/z+xwPEce6CiGwUw6IKtPOthwZ1nbGdowsislEMiyoglRbOXdz54wFOXU61djlERFWOYVFF2vt6oUFd58Irozi6ICIbw7CoIlJp4dzF7VSOLojI9jAsqlD7Fl6o7+nMK6OIyOYwLKrQX6OLLJxO+MPa5RARVRmGRRVr71cPXh5OnLsgIpvCsKhiDlIp+ndujFspWThzhaMLIrINDAsL6ODnhXpFowsbuAM8ERHDwhIcpFL079QYN5OzcOY3ji6IqPZjWFhIR38v1HN3wtbDHF0QUe3HsLAQB6kU/YpGF2d/S7N2OURElcKwsKCQAC+o3ZXYyrkLIqrlGBYWVDy6uHE3E2evcnRBRLUXw8LCQvzr4wk3JbZx7oKIajGGhYXJHApHF4l3M3H+GkcXRFQ7MSyqQaeAwtHF1sOJHF0QUa1k0bC4du0afHx8Sv1s2rQJAHDx4kWMHDkSQUFBCA0NRXR0tCXLsZri0cX1pAycv3bP2uUQEVWYzJJPfvnyZbi6uuL77783alepVLh37x5Gjx6NsLAwzJs3D+fOncO8efOgUqkwdOhQS5ZlFZ0C6mP7kURsO3IdgU09IZFIrF0SEZHZLBoWCQkJaNasGdRqdaltn332GeRyOebNmweZTIZmzZrhxo0bWLVqlU2GhcxBiohOjfD595dx4fo9BDSta+2SiIjMZtHTUJcvX0azZs3K3BYXF4e2bdtCJvsrrzp06IBbt24hOTnZkmVZTZfABqhbR4HYw9eRnpmHvHwd5zCIqFaw+MiiUaNGGD58OG7evInGjRtj8uTJ6NKlC5KTk/H0008b7V+vXj0AQFJSEry8vCxZmlUUji4a4/PvL2PmR0cAAA5SCVyUMjgp5XBRyuCskMFZKYNzicdOShlclPIS2wofOykc4CDlNQpEZHkWC4vs7Gzcvn0bnp6emDlzJlxcXLBt2zaMHz8eMTExyM3NhaOjo9ExxY/z8vIq9Fp167o+dp1qteqxj30cg3v5oIm3B1Lv5yArW4sHOfnIysk3+m9a6gPDY72+/JGHk0IGFyc5XJ3kxv91lsNVKYeLc1GbUg5XZ0c8KMgw7KN0dLCbuZPq/nOuCdhn21ed/bVYWDg7O+PkyZOQy+WGEAgICMDVq1exevVqKJVKaLVao2OKHzs7O1fotdLSskx+qJZFrVYhNTWzwsdVVqMnnNHoCdN9FEIgL1+H7NyCwp+8AjzIzS/1OCe3AA+KHt/JzC1qL0CeVlfu8ztIJYVho/xrNONc9PhRo5nikY6zQgaZQ+0Y1Vjrz9ma2GfbV5n+SqWSCv+SbdHTUC4uLqXaNBoN9u/fj4YNGyIlJcVoW/Hj+vXrW7KsWkMikUDpKIPSUQbPOhU/XqfXIydPZxQwMoUMSSmZRgGTXbT9QW4B/vgz1/BYZyKAFXKHv4JEURQ2jziVVvKxk0JmV6MaIltgsbA4ffo0xo4di7Vr16Jly5aG9vj4eDRv3hyBgYH44osvUFBQYJjkPnbsGBo3blzm1VNUcQ5SKVydpHB1khvazP1tRAgBbYG+KGTySwVLWSOdtIxc3ErJR3ZeAXLyyh/VSCWSh4LFeGRjaqRTW0Y1RLbCYmEREBAAb29vzJkzB3PnzoW7uzs2bNiA06dPY+PGjfDy8sLq1asxe/ZsTJgwAfHx8fjss8/w1ltvWaokqgCJRAKF3AEKuQM8VIoKH6/Xi8JwMREwJU+l3cvIM+xfoCt/VOMolxqdFnNRPDpgnvwzD9pcreFUGkc1RBUnERa8djM5ORlRUVE4cuQIMjIy4O/vjxkzZqB9+/YAgPPnz2PBggW4cOEC1Go1Ro8ejVGjRlX4dWrbnIU11ZY+a/N1hrmXwlNm+UVB8vBIpyiMDP9fgJy8ApT3t0EigdGpMaPRjEJeYqRT+PjhkY5cVvNHNbXlz7kq2Vufq3vOwqJhUV0YFuazhz7r9QK52qI5mdwCyJVyJCVnGB5n5xmPbLIfCqP8An25z+8ok5Z9AYBCbpj8d1HKoLDiCKZOHSUyMnKt8trWYm99buztDrWro+kdy1DjJriJrEEqlRRNthfO1ajVKjzprjT7+PwCXYlTZMajl7JGOn9maZGU9sAQQLX+ty+qFWQOEiz/dzcoHB2q5/Wq5VWIahG5zAFurg5wc32MuRohkJunQ3ZevslLly3Jw9MF6fceWO31rcHe+tzI2wMFefnV9noMC6IqZLjKS2ndf1pqtQrODvY1iW9vffaoo0RqavWFRc2fqSMiIqtjWBARkUkMCyIiMolhQUREJjEsiIjIJIYFERGZZBOXzkqlj3+5XGWOra3YZ/vAPtu+x+3v4xxnE7f7ICIiy+JpKCIiMolhQUREJjEsiIjIJIYFERGZxLAgIiKTGBZERGQSw4KIiExiWBARkUkMCyIiMsnuwkKv12PZsmXo2rUrWrVqhbFjx+LGjRvWLqtKZWVlYeHChejRoweCg4MxcOBA7N2717D99u3bmDhxIlq3bo1OnTph8eLFKCgosGLFVef69esIDg7Gpk2bDG223N/Y2Fj07dsXgYGBiIiIwK5duwzbbLHfWq0WUVFRCA0NRXBwMF566SWcOnXKsN2W+rxy5Uq8+OKLRm3m9O+LL75Az5490bJlSwwbNgznzp2rmoKEnVm2bJno2LGjOHDggLh48aIYP3686Nmzp8jNzbV2aVVm6tSpIiwsTBw5ckQkJiaKjz/+WPj6+oqjR4+KvLw8ER4eLiZOnCguX74sfvzxR9G+fXuxZMkSa5ddaVqtVgwcOFBoNBqxceNGIYSw6f7GxsaKFi1aiM8++0wkJiaKFStWCF9fXxEXF2ez/V6yZIkICQkRBw8eFNevXxdvvPGGCA4OFnfv3rWpPq9fv174+PiI4cOHG9rM6d8333wjWrVqJbZt2yauXLkiZs2aJdq1ayf++OOPStdkV2GRl5cngoKCxPr16w1tmZmZolWrVuLbb7+1YmVVJyUlRWg0GrF//36j9lGjRokZM2aIbdu2CX9/f3H//n3Dto0bN4qgoCCRk5NTzdVWraioKDFy5EijsLDV/ur1etG9e3exYMECo/axY8eKDz/80Gb7HRERId555x3D48zMTKHRaMSOHTtsos93794VEydOFEFBQeLZZ581Cgtz+hceHi7effddw/aCggLRvXt38eGHH1a6Nrs6DXXx4kVkZ2ejY8eOhjZXV1f4+fkhLi7OipVVHScnJ3z66ado27atUbtEIsGff/6JuLg4tGjRAm5uboZtHTp0QHZ2Ni5cuFDd5VaZEydO4Ouvv8a7775r1G6r/b127Rru3LmDfv36GbVHR0djypQpNttvT09P7N+/H7dv34ZOp8OmTZvg6Oho+Ddc2/t84cIFuLi4YNu2bWjVqpXRNlP9++OPP5CYmIgOHToYtjs4OKBNmzZV8vlmV2GRnJwMAPDy8jJqr1evHpKSkqxRUpVzdXVFt27d4Orqamg7c+YMjh07htDQUCQnJ6N+/fpGx9SrVw8AcPfu3WqttapkZGQgMjISb775Jho0aGC0zRb7CwCJiYkACs/hT5gwASEhIRgyZAj27dsHwHb7PWfOHMhkMvTs2ROBgYGIiorC0qVL0bhxY5voc48ePRAVFYWGDRuW2maqf8Wfb2XtUxWfb3YVFjk5OQAAR0dHo3ZHR0dotVprlGRxV69exdSpU9GqVSsMGzYMubm5ZfYfAPLy8qxRYqXNmzcPQUFB6N+/f6lttthfoPAiBgCIjIxE7969ERMTgy5dumDy5Mk4cuSIzfb7ypUrcHV1xUcffYSvv/4aAwcORGRkJC5cuGCzfS5mqn+W/nyzicWPzKVUKgEU/jZW8g3VarVwdna2VlkWc+LECUydOhVPPvkkVq5cCblcDqVSWeovTvHj2vgexMbGIi4uDtu3by9zu631t5hcLgcAjBkzBoMGDQIAtGjRAvHx8YiJibHJfv/++++IjIzE6tWrDaeSAwMD8dtvv2HZsmU22eeSTPWv5Ofbw/tURf/tamRRfIoiJSXFqD0lJaXUqanabtu2bRgzZgz8/f2xbt06uLu7AygcopbV/+Jttc0333yDtLQ0w6WUwcHBAID//ve/iIiIsLn+FiuuXaPRGLU3b94ct2/ftsl+nzt3Dvn5+QgMDDRqb9WqFRITE22yzyWZ6t+TTz5p1FZyn6r4fLOrsPD19YWrqyt++eUXQ1tWVhZ+/fVXtG/f3oqVVa3t27cjMjISffr0wcqVK43mL9q1a4eLFy8iIyPD0Hb8+HG4uLjAz8/PGuVWynvvvYedO3ciNjbW8AMAU6dOxapVq2yuv8X8/Pzg4uKC8+fPG7UnJCTgqaeessl+F3/gX7582ag9ISEBTZo0sck+l2Sqf56enmjSpInR55tOp8PJkyer5vOt0tdT1TJLliwR7du3F3v27DF8zyI8PFzk5eVZu7QqkZSUJFq1aiVGjRolkpOTRUpKiuEnPT1d5Obmil69eomxY8eKixcvir1794oOHTqIZcuWWbv0KlPy0llb7u9HH30kgoODxdatW8WNGzfEihUrhI+Pjzh69KhN9lun04kXX3xR9O7dW/z888/i+vXrYunSpcLX11ecPHnS5vo8a9Yso0tnzenf119/LQIDA8U333wjrly5Il577TXRvn17kZaWVul67C4sCgoKxOLFi0VISIgICgoS48aNEzdv3rR2WVVm7dq1QqPRlPlT/BcvMTFRjB07VgQGBorOnTuLJUuWCJ1OZ+XKq07JsBDCtvu7Zs0a0bNnT+Hv7y/69+8vfvjhB8M2W+z3/fv3xbx580RoaKgICgoSw4YNEz///LNhuy31+eGwEMK8/kVHR4tu3bqJli1bihdffFHEx8dXST0SIYSo/PiEiIhsmV3NWRAR0eNhWBARkUkMCyIiMolhQUREJjEsiIjIJIYFERGZxLAgu/Paa68hOjq63H22bNmCiRMnVlNFhW7fvm24XQlRTcOwICIik+zqrrNkP/R6PRYuXIizZ8/iwYMHEEJg/vz5aNOmjdF+fn5++Oc//4lDhw4hOzsbM2bMQHh4OAAgNTUVEyZMQFJSEhwcHBAVFYVmzZrhzJkzWLx4MbRaLVJTU9GpUycsXLjQ6HmvX7+O4cOH49ChQ3B0dIROp0NoaCg+++wzZGZmmjx++fLlSE9Px9y5c0s9zszMxIIFC5CQkID8/HyEhIQgMjISMpkMy5Ytw549eyCXy+Hh4YF33nnHsOYBUWVwZEE26ezZs0hJScHXX3+NnTt3YsCAAfj0009L7afT6eDk5IQtW7bg/fffx+zZs3Hv3j0AwK1bt/DGG29g+/btaNu2reHU1eeff45p06Zh06ZN2LFjB/bt24f4+Hij523SpAmaN29uWIzo8OHD8Pb2RrNmzcw6vjwLFy6Ev78/tmzZgtjYWKSnp2PNmjVISkrC2rVr8c0332DLli3o3Lkzzp0797hvIZERjizIJgUHB8PNzQ1fffUVbt26Zbg7Z1lGjBgBoPCuxBqNBidOnAAAtGzZEo0aNQJQuFbEnj17AACLFi3CwYMH8cknn+DatWvIy8tDdnZ2qecdPHgwvv32Wzz77LPYsmULhg4dWu7xxbeRN+XAgQM4f/48Nm/eDKBwURygcAVIX19fDBgwAN26dUO3bt0QEhJi7ltGVC6GBdmkAwcOYMGCBRgzZgx69uyJpk2bYtu2bWXu6+DgYPh/vV5veCyT/fXPQyKRoPg2aiNGjICPjw+6du2KPn364OzZsyjrFmt9+vTBokWLcPXqVZw4cQKLFi0y+/iSrwcA+fn5RjV+8MEHaNasGYDCZWUlEgmkUinWr1+P8+fP4+eff8bChQvRtWtXREZGVui9IyoLT0ORTTpy5Ai6d++Ol156CQEBAfjxxx+h0+nK3Ld4DYwLFy7g+vXraNeu3SOfNyMjA+fPn8crr7yC8PBw3L17Fzdv3oRery+1r0KhQEREBF577TWEh4fDycnJ7OM9PDxw4cIFCCGQlZWF/fv3G7Z16dIFn332GYQQ0Gq1mDRpEtavX49Lly6hX79+aNasGSZOnIjRo0eXWu+C6HFxZEE2afjw4Zg5cyb69++PgoICdO7cGT/88EOZH+qnTp3Cxo0bodfrsXTpUri5uT3yeevUqYMJEyZgwIABcHZ2hpeXF1q3bo0bN26UecpnyJAhWL9+PebNm2fy+IYNGxqOe+6553Do0CGEh4fDy8sL7du3N4w03njjDSxYsAD9+/dHfn4+OnXqhPHjx0Mul6NPnz4YNGiQYZnNN998s5LvJFEh3qKc7JqPjw9+/vlneHp6WrsUohqNp6GIiMgkjiyIiMgkjiyIiMgkhgUREZnEsCAiIpMYFkREZBLDgoiITGJYEBGRSf8PqYrrXzB95JQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "BernouliNB(train_tfidf,train_y,cv_tfidf,cv_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best roc auc score is 77.59792702379485\n",
      "Confusion Matrix of test set:\n",
      " [ [TN FN]\n",
      " [FP TP] ]\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEFCAYAAADpIfy5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XlcVdXex/EPg8gFQkUZNEsxQsDEFIUm0zS93dTnRhY+zoaWpaDhVKmpKXofLxJllkNKUpnoLSLNvJZlt1nEJkIQzXBE0MwQGY7Cef7weurEIKSI7vN999p/nLXWXnvtl69+rPNb6+xtZzabzYiIiOHYN/QARESkfijAi4gYlAK8iIhBKcCLiBiUAryIiEEpwIuIGJQCvIiIQSnAi4gYlAK8iIhBKcCLiBiUAryIiEEpwIuIGJRjQ178eo+ODXl5uQL9XHqqoYcgV6jTxbkXdf6Z4/tq3bZRi3YXda0rRYMGeBGRy6aivKFHcNkpwIuIbTBXNPQILjvl4EXENlRU1P6og9OnTzNv3jx69OhBSEgI48aN48CBA5b6rKwshg8fzs0330zPnj1ZtWrVH4ZVweLFi+nevTudOnUiMjKS/fv3W7W5UB/VUYAXEZtgNlfU+qiLxx9/nC1btjBr1izWr1+Ph4cHgwcP5pdffuHEiROMGjWKNm3a8NZbbzFx4kQWL17M+vXrLee/+OKLrF27ltjYWNatW4eDgwOjR4+mrKwMoFZ9VEcpGhGxDeVnL3mX2dnZfPLJJyxdupRevXoB8Mwzz7Bjxw7eeOMNHBwcaNSoEXPmzMHR0ZEbbriB/fv3s2LFCiIiIjCZTCQmJjJlyhR69OgBQEJCAnfccQebN2/mvvvuY/369TX2URPN4EXENlSU1/6opdzcXAC6detmKXNwcCAgIIC0tDTS09Pp2rUrjo6/zaXDwsI4ePAg+fn5ZGVlUVxczC233GKpd3NzIygoiPT0dIAL9lETzeBFxDbUIfVSWFhIYWFhpXJ3d3fc3d0tnz09PQE4cuQI7du3t5QfOnQIk8kEgJ+fn1UfXl5eAOTl5VFQUACAt7d3pTZ5eXkA5Ofn19jHH8/9PQV4EbENdVg8TUpKYsmSJZXKo6KiiI6OtnwODg7Gz8+POXPmEB8fT4sWLXj99dfJzs6mdevWVFRU4OTkZNXH+c9lZWWUlJRYlf2+zfk/EKWlpTX2URMFeBGxCXVZPB05ciTh4eGVyn8/ewdo1KgRS5Ys4cknn+Suu+7C0dGRnj178sADD/DDDz9gMpksgfq8859dXFxwdna2lP0+iJtMJlxcXABwdnausY+aKMCLiG2owwz+j6mYmvj6+rJu3Tp+/fVX7OzscHd3Z+LEibRt25bCwkJLGua88599fHwwm82WMjc3N6s259MyPj4+NfZREy2yiohtKD9T+6OWioqKGDZsGBkZGTRp0gR3d3eKior44osv6N69O926dWPnzp2cPfvbDp6vvvqKtm3b4unpSUBAAG5ubqSlpVn1uWvXLkJDQwEu2EdNFOBFxDaYK2p/1JKbmxt2dnYsWLCA3bt3k52dzaOPPkqrVq3o378/AwcOpKSkhOnTp7N3715SU1NZvXo1Y8eOBc7l0ocNG0ZCQgJbt24lOzubmJgYvL296du3L8AF+6iJnfn8d4QGoIeNyR/pYWNSnYt92FhZ5oe1btu4Q+9aty0oKCA2NpYvv/wSe3t77rrrLqZNm4aHhwcAGRkZzJ8/n8zMTDw9PRk1ahQjRoywnF9eXk5CQgIpKSmUlJQQEhLC7Nmzue666yxtLtRHdRTg5YqiAC/VuegA/8MHtW7b+KY+F3WtK4UWWUXENtTxGTNGoAAvIjbBXFH7xVOjUIAXEdugGbyIiEHZ4PPgFeBFxDbojU4iIgalGbyIiEEpBy8iYlD18MKPK50CvIjYBs3gRUSMyWzWIquIiDFpBi8iYlDaRSMiYlCawYuIGJR20YiIGJRSNCIiBqUUjYiIQSnAi4gYlFI0IiIGpUVWERGDUopGRMSglKIRETEozeBFRAxKAV5ExKDM5oYewWWnAC8ituGsdtGIiBhTPSyybt++nREjRlRZ17p1az788EPi4+NZsWJFpfrMzEwcHc+F4DVr1pCYmMixY8cIDAxkxowZBAcHW9oeOnSIefPmsWPHDpydnQkPDycmJsZyfnUU4EXENtRDDr5z58589tlnVmU5OTk88sgjjB07FoDdu3cTERHBhAkTrNqdD84pKSnExcUxb948AgMDWblyJWPGjGHz5s00b94ck8nE6NGj8fX1JTk5mYMHDzJ9+nQcHR2JiYmpcXz2l/BeRUSuXGZz7Y9acnJywtPT03I0bdqUBQsW0KdPHyIiIoBzAT8oKMiqnaenp6WP5cuXM2TIEAYMGICfnx/z58/Hzc2N5ORkALZs2cLhw4dZuHAh/v7+9O7dmylTpvDqq69SWlpa4/gU4EXENlRU1P74k1577TXy8vJ46qmnACgsLCQvLw8/P78q2x8/fpzc3FzCwsIsZQ4ODoSEhJCeng5Aeno6gYGBNGnSxNImLCyM4uJiMjMzaxyPUjQiYhvqELgLCwspLCysVO7u7o67u3uV55SUlLB8+XJGjBiBt7c3cG72DrBx40ZmzJjBmTNnCA0NZfLkyXh5eZGfnw+Aj4+PVV9eXl5kZGQAkJ+fX2U9wNGjR2u8DwV4EbEJ5vLav3Q7KSmJJUuWVCqPiooiOjq6ynPeeecdysrKrBZdzwd4Nzc3Fi9ezLFjx0hISGD48OGkpqZSUlICnEv1/J6TkxMmkwmA0tJSXF1dK9UDlJWV1XgfCvAiYhvqMIMfOXIk4eHhlcqrm73DuQDfp08fPDw8LGWDBw+mX79+lvRKQEAA/v7+9OjRg61bt+Lr6wtgCebnmUwmXFxcAHB2dq6yHrC0qY5y8PWgzz092bX/K6uyxs6NmTojmk/SN5F1YDvvfbyeAeF/tdTHPPEYB05kVHl89s1mAOKXxFbbJvmdVZf1HuXPsbe3JyZmLN9nfEx+QSYf/yeVHj1utWozddp4snd/zrHjWWzc+Br+/jdY1bdr14Z161/maP4P5O7fyUtLF+Lh0fRy3sbVyVxR68Pd3Z3WrVtXOqoL8CdOnODbb7+lf//+VuV2dnZWuXMAb29vmjZtSl5eHq1atQKgoKDAqk1BQYElzePj41Nl/fm6mmgGf4mFhHbiueX/wM7Ozqp8waKn6dvvLhbNX8KPe36izz09eXHVIsxmeDd1C2tfS+HjDz+3OueGG31Z9MJckl9LAWDxomW8vnq9VZtbb+/Gk7Mft7SRK1tMzFhmzZ7MvHnPsjP9O0aMiCD1nSR69gjnu+8yeWr6RCZPfoynZ/4f+w8c4oknotj03hpCuvShsPAUHh5Nef+D9ZSWljFxwgwKTxXxxBPRbN68ljvu+B/OnDnT0Ld45aqov1+yfv3119jZ2dGtWzer8tjYWNLT00lNTbWUHTx4kF9++QU/Pz88PDzw9fUlLS2N7t27A1BeXs7OnTsZNGgQAN26dSMlJYXCwkLLH5jt27fj6upKUFBQjeOyM5sb7ve713t0bKhLX3JOTo2IHDuMydOjKCkuoVGjRgRef25l3KN5M77d8wlTJ8xi3etvW855JflFmjdvxv/0GVKpP3t7ezZ88AZFRaf537+PrvKabte48sHnb/Plp2lMGj+zfm7sMvu59FRDD6Fe7fx6K1/v/I6HH54MnPt33pX1KZs2bWX2rH+y98ft/HPhEp59dhkATZu6k5X9OQvmP8cLL6xiwoQxzF8wna4hfdi9+0cAmjdvRuauT3n66f/j5RWvN9i91bfTxbkXdX7xC+Nq3dYl+qU69b148WI2bdrEli1brMrT09MZOXIkgwcPZtiwYRQUFDB//nycnJxYt24d9vb2rF+/ntjYWObMmUNwcDCrVq3io48+YvPmzXh4eFBWVkb//v25/vrrmTp1KkeOHGH69OkMHTq02vWA85SiuUR63t2d8TFjWDA7ntUvv2FV5+bmymuJ6/hk2xdW5fv25nJdm2ur7G/w8IEE3uTPrCcWVHvN8Y+Pwe0aV+bPfvbib0Aui8aNnSg8VWT5XFFRQeGvp/Bo1pTQ0M5cc40bmzZ9YKk/ebKQzz7bTp8+PQDwu7EdBw8esQR3gJ9//oWcnB8tbaQa5eW1P+ro2LFjlVIxAF27dmXZsmVkZGQQHh5OdHQ0HTp0YMWKFdjbnwu/ERERPP744zz//PMMHDiQ/fv3k5iYaMnlN27cmJUrV1razpo1i0GDBjF+/PgLjuuCKRqTycS///1v0tPTycvLo6ysDBcXF3x8fAgNDaVv374X/LmsLfjumx+4/eZ7KCw8RcwTj1nVHdh/iBlTYq3K7O3t6Xn3Hfy456dKfTVu7ETME4+xfk0qOdk/VqoH8PRqzuhHh/F83DJ+Pn7i0t2I1KsVy1/lyacmsHHDFr7+OoNhwx4gMMifOc8swu/Gcwtu+/YdsDon96eD9OvfB4BDh47QvHkznJ0bU1p6bgeFg4MDrVu3ovEfdmLIH9Tj0yTnzZtXbV337t0t6ZfqREZGEhkZWW19mzZtWLWq7utsNc7gDxw4QL9+/Zg9ezY//vgjbm5utGzZkr/85S/s2bOHGTNmMGDAAA4dOlTnCxtNfl4BhYW1Ty9MenI8N/q3Y9niVyrVDbj/bzT39GD5ktXVnj/soQjOni3ntcT11baRK8/LL7/Ol1+ms+m9N8g7mkHcotnMnRvPe5u24n7NNZSWllXKo58qKuKaa9wAeDvlPRwdHVm5KoHrrrsWb29Pnnt+Hk2auOPiWvOOCptXYa79YRA1Tr2feeYZfH19efvtt3Fzc6tUX1RURExMDPPmzWP58uX1NkijeWxCJBOmPMLyJavZuuU/leqHjHyAj7d+Ru4fZnK/97/DB/Jm8oY6/VGRhvfOhlcJCLiRiRNnsjt7L3f1up3p0yfy66+FYAdVLYnZ2dlR8d/Z5549+xg1agIvvLCA7N2fU15ezhtvpLBp0wcEtK/615LyX3qjk7WdO3eyfv36KoM7nNu8P3nyZIYMqbxIKFV7OnYqD48bQdLKZObPiq9U38KzOV26BhPz2Ixq++jU5SZatvJmY8rm+hyqXGK33tqV228PZdjQcbz99nsAfPrpVzg6OBIb+xRzZsfRuLETjo6OnP3do23dXF2t/pBv3LCFTe9+QLt2bTh58leOHz/Be5vXcuKXXy/7PV1VDDQzr60aUzTu7u6Wn9JW5/DhwxfcbC/nZmHPLV3Aw+NG8EL8Cp6eNr/Kdj163UZ5eTkf/HtbtX317H0HBfnH2bH9m/oartSD1q3P7XlOS7P+d/viyx24urpgNpuxt7enbdvrrOrb+l7Hnj37ALjuumsZMeJBKioq2Lv3J44fP4GdnR1BQf5kfL/r8tzIVcpcUVHrwyhqDPAPPPAATz75JMnJyezbt4/i4mLOnj1LcXExubm5rF+/nhkzZnD//fdfrvFetZ6Oncr9gwYwd2YccfNfqLZdpy4d2ZvzE0WnTlfb5uYuN/Hd1xn1MUypR3v2ngvSt97a1aq8W7ebOXPmDO9s+DclJaUMGNDXUte0qTt33BHGx9vO/UaiVStvli6L4+abO1jaDBzYH0/P5rz33tbLcBdXsXrcRXOlqjFFEx0djZ2dHf/85z8tz0z4PVdXV4YOHcrEiRPrbYBGcFNwIJFjh/LJti/YmfYtnbv+9iD/8vJyvv/mtyfCtQ/0Y9/e3Br78w/0Y9M779fXcKWefPvND2ze/CEJz82jmUcTdmf/SPc7b2HSpEd56cVXOHL4KMuWrmbW7MlUVFSwZ+9PTJsWxalTRaxevQ6AHTu+5dtvf+Clpf9kzuw4fFp6ERc3my1btrFt2+cXGIGNs8EUTY0B3s7OjujoaMaOHUt2djb5+fmUlJTg7OyMj48PAQEBlR6SI5X1+VtP7O3tufOu27jzrtus6k4XFVt+EAXQwtOD3H37a+yvRQsPCn/V4urVaNjQccyePYVp06Jo1qwpP+79iSlTnmHVyjUAzJ4dR0WFmYmPP4Krqwvbv9rJIw9PtuTgKyoqGBTxCPHPPsPqpMWcPl3MK4lreeaZRQ15W1cHA6Veaku/ZJUritF/ySp/3sX+kvX0rP+tdVvXuckXda0rhX6hJCK2QdskRUQMSjl4ERFjMp81zu6Y2lKAFxHboBm8iIhBKQcvImJQmsGLiBiTWQFeRMSgtMgqImJQmsGLiBiUAryIiDE14FNZGowCvIjYBs3gRUQMSgFeRMSYzGf1QycREWOyvfiuAC8itkE/dBIRMSobDPA1vnRbRMQwKupw1FFqair33nsvHTt2pF+/fmzevNlSd+jQIcaOHUuXLl247bbbiIuL4+zZs1bnr1mzht69exMcHMygQYP4/vvvrepr00dVFOBFxCaYK8y1PurinXfeYfr06QwaNIh3332X/v37M2nSJHbu3InJZGL06NHY2dmRnJzMvHnzePPNN3nhhRcs56ekpBAXF8fjjz9OSkoKvr6+jBkzhp9//hmgVn1UR+9klSuK3skq1bnYd7KeCO9R67Yeb/+nVu3MZjO9e/fm7rvvZvr06Zby0aNH06VLF66//nqeeuopPv/8c5o0aQLAv/71LxYsWMCXX36Js7Mzf/3rX+nduzfTpk0DoLy8nD59+jBw4EDGjx/Pxo0bL9hHdTSDFxHbUA8pmn379nH48GH69+9vVb5q1SrGjx9Peno6gYGBlsAMEBYWRnFxMZmZmRw/fpzc3FzCwsIs9Q4ODoSEhJCeng5wwT5qogAvIjbBXFH7o7Zyc3OBc2mURx55hFtvvZUHH3yQjz76CID8/Hx8fHyszvHy8gLg6NGj5OfnA1TZJi8vr1Z91ES7aETENtQhcBcWFlJYWFip3N3dHXd3d8vnoqIiAKZNm8b48eOJiYnh/fffZ9y4caxatYrS0lJcXV2t+nBycgKgrKyMkpISq7LftzGZTAAX7KMmCvAiYhPqMjNPSkpiyZIllcqjoqKIjo62fG7UqBEADz30EAMHDgQgMDCQH374gcTERJydnS2B+rzzn11cXCz586rauLi4AFywj5oowIuITTBfeFehxciRIwkPD69U/vvZO/yWWvH397cqv/HGG/nwww+59dZbycrKsqorKCiwnNuqVStLWfv27a3aeHt7W9rV1EdNlIMXEZtQlxy8u7s7rVu3rnT8McAHBQXh6upKRkaGVXlOTg7XX3893bp1Iysryyrds337dlxdXQkKCsLDwwNfX1/S0tIs9eXl5ezcuZPQ0FCAC/ZREwV4EbEJ9bHI6uzszJgxY3jppZfYsGEDBw4cYOnSpXz22WdERkZy99134+3tTUxMDNnZ2Xz00UfEx8fz0EMPWfLokZGRJCUlkZKSwt69e5k5cyanT5/mwQcfBKhVH9XRPni5omgfvFTnYvfB5/fsWeu23h9/XKe+V69ezeuvv87Ro0dp164d0dHR9OnTB4D9+/czd+5cduzYgbu7OwMHDmTixInY2/82v05MTCQpKYmTJ0/SoUMHZsyYQYcOHSz1temjKgrwckVRgJfqXGyAP3pnz1q39fnk44u61pVCi6wiYhPMFXYNPYTLTgFeRGxCRbkCvIiIIdVl8dQoFOBFxCYoRSMiYlANt52k4SjAi4hN0AxeRMSgtMgqImJQmsGLiBiU2awALyJiSNomKSJiUBWawYuIGJNSNCIiBqVdNCIiBqVdNCIiBqUcvIiIQSkHLyJiUHoWjYiIQSlFIyJiUBVaZL28jhSdaMjLyxWo5MinDT0EMSjN4EVEDEqLrCIiBqUZvIiIQdngJhoFeBGxDeUV9g09hMtOAV5EbIINPi1YAV5EbIMZ28vB2953FhGxSRXm2h9/1k8//UTnzp3517/+ZSmLj4+nffv2lY6zZ89a2qxZs4bevXsTHBzMoEGD+P777636PXToEGPHjqVLly7cdtttxMXFWZ1fHc3gRcQmVNTzDP7MmTNMmTKF4uJiq/Ldu3cTERHBhAkTrModHc+F35SUFOLi4pg3bx6BgYGsXLmSMWPGsHnzZpo3b47JZGL06NH4+vqSnJzMwYMHmT59Oo6OjsTExNQ4Js3gRcQmmLGr9fFnvPDCC7i6ulYqz8nJISgoCE9PT6vjvOXLlzNkyBAGDBiAn58f8+fPx83NjeTkZAC2bNnC4cOHWbhwIf7+/vTu3ZspU6bw6quvUlpaWuOYFOBFxCaUY1fro6527NjBunXrWLhwoVV5YWEheXl5+Pn5VXne8ePHyc3NJSwszFLm4OBASEgI6enpAKSnpxMYGEiTJk0sbcLCwiguLiYzM7PGcSlFIyI2oS67aAoLCyksLKxU7u7ujru7e6W206ZNY+bMmbRs2dKqLicnB4CNGzcyY8YMzpw5Q2hoKJMnT8bLy4v8/HwAfHx8rM7z8vIiIyMDgPz8/CrrAY4ePVrjfSjAi4hNqEuAT0pKYsmSJZXKo6KiiI6OtiqbM2cON998MwMGDKjU/nyAd3NzY/HixRw7doyEhASGDx9OamoqJSUlADg5OVmd5+TkhMlkAqC0tLRS6ud8+7KyshrvQwFeRGxCXXLrI0eOJDw8vFL5H2fvqamppKens3Hjxir7GTx4MP369bOkVwICAvD396dHjx5s3boVX19fAEswP89kMuHi4gKAs7NzlfWApU11FOBFxCbU5WnBVaViqvLWW2/x888/07NnT6vyuXPnsnr1ajZt2mSVOwfw9vamadOm5OXlcfvttwNQUFBA+/btLW0KCgrw9vYGzqVvsrKyrPooKCiw1NVEAV5EbEJ9bJNctGhRpZ0sffv2JSoqiv79+xMbG0t6ejqpqamW+oMHD/LLL7/g5+eHh4cHvr6+pKWl0b17dwDKy8vZuXMngwYNAqBbt26kpKRQWFho+aOzfft2XF1dCQoKqnF82kUjIjahvA5HbXl7e9OmTRurA8DDw4Nrr72We+65hz179hAbG0tubi5paWlERUURHBxsmfVHRkaSlJRESkoKe/fuZebMmZw+fZoHH3wQgLvvvhtvb29iYmLIzs7mo48+Ij4+noceeqhS7v6PNIMXEZtQYXf5H1XQtWtXli1bxpIlSwgPD8fJyYnevXszdepU7O3Pza8jIiIoKiri+eef5+TJk3To0IHExEQ8PDwAaNy4MStXrmTu3LlERETg7u7OoEGDGD9+/AWvb2c2N9yraB2drm2oS8sVSm90kuo0atHuos7/V8uhtW77YN6ai7rWlUIzeBGxCXqapIiIQdngO7cV4EXENvyZRxBc7RTgRcQmaAYvImJQysGLiBiUXrotImJQStGIiBiUUjQiIgZVrhm8iIgxaQYvImJQCvAiIgalXTQiIgalXTQiIgalFI2IiEHV5UUeRqEALyI2QSkaERGDUopGRMSgtItGRMSgKmwwxCvAi4hN0CKriIhBKQcvImJQ2kUjImJQysGLiBiU7YV3BXgRsRG2mIO3b+gBiIhcDuWYa33URX5+PpMmTSIsLIzOnTvzyCOPsGfPHkt9VlYWw4cP5+abb6Znz56sWrXK6vyKigoWL15M9+7d6dSpE5GRkezfv9+qzYX6qI4CvIjYhIo6HLVlNpt5+OGHOXr0KKtWreLNN9/E2dmZUaNGcfr0aU6cOMGoUaNo06YNb731FhMnTmTx4sWsX7/e0seLL77I2rVriY2NZd26dTg4ODB69GjKysoAatVHdZSiERGbUB+LrMePH+eGG25gwoQJ+Pr6AjBu3Dj+/ve/k5OTw/bt22nUqBFz5szB0dGRG264gf3797NixQoiIiIwmUwkJiYyZcoUevToAUBCQgJ33HEHmzdv5r777mP9+vU19lETzeBFxCaY63DUlqenJwkJCZbgfvz4cVatWoWXlxf+/v6kp6fTtWtXHB1/m0uHhYVx8OBB8vPzycrKori4mFtuucVS7+bmRlBQEOnp6QAX7KMmmsHXEw+PZhQc/aFS+Vspmxj0v4/g6dmcRXGzufdvvQH4aNvnTHtiLvv3HwJg1tOTmPX05Cr7/umnA9zY/tb6G7xcEsXFJSQsfYUt2z6ltLSMmzsGMmncaAJubAfAqaLTPLfsFT785AuKi0sJC+nE1OiHub51qyr7W/j8cvYfOsJLcc9YymbExvPO5q1Vtu/WOZhXliy89Dd2lapL6qWwsJDCwsJK5e7u7ri7u1d5zpNPPsnbb7+Nk5MTS5cuxdXVlfz8fPz8/KzaeXl5AZCXl0dBQQEA3t7eldrk5eUBXLCPP577ewrw9aRTcBAAf7t3MIWFRZbyn0/8QqNGjXh/yzp8vL14avoCDh48TNT4SD75OJXOIX04ceIXViWuZcuWj636bN/ej5Uvx5P4ytrLeSvyJz0+I5ZvMnYxLnIo/jf4sun9bYwcN4XklYvxbdOaaXMWkp3zI5PGjaap+zUsX72Wh6Kf4J3Xl+Hm6mrV1xtvbuC19anceVuoVfnYUYOJuO9eq7Id32Tw3LJXGDjgr/V+j1eTuiyeJiUlsWTJkkrlUVFRREdHV3nO6NGjGTp0KG+88Qbjx49nzZo1lJaW4uTkZNXu/OeysjJKSkqsyn7fxmQyAVywj5oowNeTjh0DOXq0gA+2flKpLjz8XjreFMi9/Ybw/gf/AWDbx1+w64dPmDZ1HE8+NZ/Dh/M4fDjPco69vT3PPx/LJ598xT/+b/Fluw/5czKz9/BF2tfMmhptCcC3h4UwdOwklqx8lXGRQ/n0yx0kzJ9Jn563A+DXrg19B45i22fbGfDXXgD8/MtJEl5KZMO/P+QaN9dK17m+dSurGX/R6dNMnvUP/uee3vT/bx9yTl1y8CNHjiQ8PLxSeXWzd4Abb7wRgPnz5/Pdd9/x2muv4ezsbAnU553/7OLigrOzs6Xs90HcZDLh4uICcME+aqIcfD3p2DGQjIysKuv8b2zH2bNn+fCjzyxlJpOJ9J3f0bdvzyrPGR05hE7BQUx4fEZ9DFcusf0HDwPngvrvde4YxOfbd9K6VUveWJHAnbd2s9Q1cmwEgMl0xlL2clIyX3+fyfJnY2n/39ROTV5+dT2nTxczOWrMpbgNQ6lLDt7d3Z3WrVtXOv4Y4AsKCti4cSNm82+0ZSiXAAANG0lEQVR/POzt7fHz8yM/Px8fHx9LGub35wD4+PjQsmVLq7LftzmferlQHzVRgK8nHTsG4uLyFz79zzsUFf5I7r50pkx+DICDh47g6OhIq1bWubO2ba+jbZvrKvXVuHFjZj09iVdWr2PXrpzLMn65OD5eLQDIy7f+H/Nw3lGKThdTWlZGcIcAGjd24uzZcn78aT9P/yOB5h7N6H3nb+srg8L7s2HNCm7t1vmC1zz+8wleX59K5NAHad6s6aW9IQOowFzro7by8vKYMmUKO3futJSdOXOGXbt2ccMNN9CtWzd27tzJ2bNnLfVfffUVbdu2xdPTk4CAANzc3EhLS7PUFxUVsWvXLkJDz6XjLtRHTRTg64GdnR1Bgf74+9/AipWv03/AcNatS2V+7FPMnPE4W7Zs49ixn0l6ZTEBAX54eDRj1tOTuKlDe1xdK3/lGhTxd7y8WvBswrIGuBv5M24K9KftddcSu+hFfsjKofBUEetT3+PTL8/tjCguKbW0nbPwef4+7FG+SPuaSY89RNMmv80Sfdu0xtHRoVbXXJf6Hg4ODgwK73dpb8Yg6mMffMeOHQkLC2PWrFmkp6eTk5PDE088wcmTJxk1ahQDBw6kpKSE6dOns3fvXlJTU1m9ejVjx44FzuXShw0bRkJCAlu3biU7O5uYmBi8vb3p27cvwAX7qImd+fffLaowZMgQ7Oxq9xi2NWvW1KrdeY5O19ap/dXC3t6eHnfeyoGDh/nxx1xL+ZIX/sHwYQ/g5XMTXUOCWf3KYnx9rwfg3U0fkJt7kJEjImjWvL1Vf5/+5x1OnDjJ38NHXs7baBAlRz5t6CFcMnv37WfaMwvJ2fsTAJ1uCuS20C4sTVzDZ++tswTyjF27KSkp5cNPv2TNv97hmScfr3KBdFTUNFz+8herXTTnmc1m7g4fTq/utzFj8rj6vbEG0qjFhVNUNRnT9oFat12Z+2at2/76668sWrSIbdu2cerUKbp27cq0adNo3/7c/8cZGRnMnz+fzMxMPD09GTVqFCNGjLCcX15eTkJCAikpKZSUlBASEsLs2bO57rrfvs1fqI/qXHCRtUePHjz33HO0a9eO4ODgWt+0LauoqGDbx59XKt/y/jYeHTsCP7+2fP7FDm5sfyu+vtdTVmbiyJGjrHz5WU6cOGl1jpdXC8LCujAqcuLlGr5cIn7t2pCS9BJ5+ccoLy+ndSsfXkpcg729vdUumY5B5wJBaEgn8guO8/KryXXeAfNDVg75x37mb3ffeUnvwUjq+giC2mrSpAnz5s2rtr5jx44kJydXW+/g4MCUKVOYMmXKn+6jOhcM8GPHjsXNzY34+HiWL19O69at63wRW9OypTf97r2b1Hc2c/z4CUv5X/5ybsX85MlCRgyP4K2Ud/nppwOW+o4dA/nu+0yrvvr27Ul5eTkbN75/eQYvl0RJaSkffPw5t4TcTEvv3/KkOXt/wq9dG/LyC9jxzfeE9+tr9Q050P8GPvkyraoua/TZV+k092hG5+AOl2T8RqSHjVVj6NChhIaG8txzz9X3eAyhcWMnli39J0OHDLQqvz+8H7tzfqSiooLEVQn06dPDUndLWAghXYJ5990PrM7p1vVmsrL3cOpUEXL1cHR0ZF7cC2z+8D+WskNHjvLplzvoeXsY+w8eZtY/nmPH199b6s1mM1/u+Job27Wt8/UysnLoGOhf63SqLaowm2t9GEWt98HPnTuXzMzMCzcUcnMPsjb5bZ6ZM5WKigqys/cwcGB/7g+/l/sfiCQvL58NG7cQt3AWZrMZJycnnl00h2+/y+S1161zfzd1aE9Ozr4GuhP5sxo5OnL/gHtYkZSMR7OmuLm48OzSRJo1a8KIQeG4ubrSqUMAM+Y/y4SxI2nWxJ2Ud7fwTcYuli6q/ut+dfbuy6XvXd3r4U6Mwzhhu/ZqHeC9vLwsP4+VC3v4kSnMnPE4E6LH0LKlF1nZe3lw0MOWGfroMZN4Nv4ZVixbREVFBZve28qTT8Vy5swZq348vVqw57+LdHJ1iXnsIeyA+BdXYjKdIbRLJyaPH21ZXH0x7hmeW/YKCS8l8mvhKYIC/Hg5YQGhIZ3qfK0Tv/zKNddU/iGU/MYW3+h0wV009cmou2jkzzPSLhq5tC52F83gNvfVuu3a/akXda0rhR5VICI24awNzuAV4EXEJpgV4EVEjMkWt0kqwIuITWjA5cYGowAvIjbBFnfRKMCLiE2or0cVXMkU4EXEJmgGLyJiUMrBi4gYlHbRiIgYlPbBi4gYlHLwIiIGVW62vSSNAryI2ASlaEREDMpIL/KoLQV4EbEJthfeFeBFxEZokVVExKAU4EVEDEq7aEREDEq7aEREDErPohERMShbzMHbN/QAREQuB7PZXOvjz1q+fDmDBw+2KouPj6d9+/aVjrNnz1rarFmzht69exMcHMygQYP4/vvvrfo4dOgQY8eOpUuXLtx2223ExcVZnV8dzeBFxCaU1/PzJNesWUNCQgKdO3e2Kt+9ezcRERFMmDDBqtzR8Vz4TUlJIS4ujnnz5hEYGMjKlSsZM2YMmzdvpnnz5phMJkaPHo2vry/JyckcPHiQ6dOn4+joSExMTI1j0gxeRGxChdlc66Mu8vPzefTRR1m0aBG+vr6V6nNycggKCsLT09PqOG/58uUMGTKEAQMG4Ofnx/z583FzcyM5ORmALVu2cPjwYRYuXIi/vz+9e/dmypQpvPrqq5SWltY4NgV4EbEJ5jr8VxeZmZm4urqyYcMGOnXqZFVXWFhIXl4efn5+VZ57/PhxcnNzCQsLs5Q5ODgQEhJCeno6AOnp6QQGBtKkSRNLm7CwMIqLi8nMzKxxbErRiIhNqMvMvLCwkMLCwkrl7u7uuLu7W5X16tWLXr16VdlPTk4OABs3bmTGjBmcOXOG0NBQJk+ejJeXF/n5+QD4+PhYnefl5UVGRgZw7htCVfUAR48erfE+FOBFxCbUZWaelJTEkiVLKpVHRUURHR1d637OB3g3NzcWL17MsWPHSEhIYPjw4aSmplJSUgKAk5OT1XlOTk6YTCYASktLcXV1rVQPUFZWVuP1FeBFxCbUZQY/cuRIwsPDK5X/cfZ+IYMHD6Zfv36W9EpAQAD+/v706NGDrVu3WnL254P5eSaTCRcXFwCcnZ2rrAcsbaqjAC8iNqEujyqoKhXzZ9jZ2VnlzgG8vb1p2rQpeXl53H777QAUFBTQvn17S5uCggK8vb2Bc+mbrKwsqz4KCgosdTXRIquI2IT6WmStSWxsLPfdd59V2cGDB/nll1/w8/PDw8MDX19f0tLSLPXl5eXs3LmT0NBQALp160ZWVpbVmsD27dtxdXUlKCioxusrwIuITTCbK2p9XCr33HMPe/bsITY2ltzcXNLS0oiKiiI4OJiePXsCEBkZSVJSEikpKezdu5eZM2dy+vRpHnzwQQDuvvtuvL29iYmJITs7m48++oj4+HgeeuihSrn7P1KKRkRsQkM8qqBr164sW7aMJUuWEB4ejpOTE71792bq1KnY25+bX0dERFBUVMTzzz/PyZMn6dChA4mJiXh4eADQuHFjVq5cydy5c4mIiMDd3Z1BgwYxfvz4C17fztyAT+BxdLq2oS4tV6iSI5829BDkCtWoRbuLOv96j461bnvgRMZFXetKoRm8iNgEW3zYmAK8iNiE8gq98ENExJD0wg8REYPSCz9ERAxKOXgREYPSDF5ExKC0yCoiYlBK0YiIGJRSNCIiBlXXV/EZgQK8iNgE7YMXETEozeBFRAyq4hI+BvhqoQAvIjZBi6wiIgZliwG+QZ8HLyIi9Uev7BMRMSgFeBERg1KAFxExKAV4ERGDUoAXETEoBXgREYNSgBcRMSgFeBERg1KAFxExKAX4BlRRUcHixYvp3r07nTp1IjIykv379zf0sOQKsnz5cgYPHtzQw5CrlAJ8A3rxxRdZu3YtsbGxrFu3DgcHB0aPHk1ZWVlDD02uAGvWrCEhIaGhhyFXMQX4BmIymUhMTCQqKooePXoQEBBAQkICx48fZ/PmzQ09PGlA+fn5PProoyxatAhfX9+GHo5cxRTgG0hWVhbFxcXccsstljI3NzeCgoJIT09vwJFJQ8vMzMTV1ZUNGzbQqVOnhh6OXMX0uOAGkp+fD4C3t7dVuZeXF3l5eQ0xJLlC9OrVi169ejX0MMQANINvICUlJQA4OTlZlTs5OWEymRpiSCJiMArwDcTZ2RmgUjA3mUy4uLg0xJBExGAU4BtIy5YtASgoKLAqLygoqJS2ERH5MxTgG0hAQABubm6kpaVZyoqKiti1axehoaENODIRMQotsjYQJycnhg0bRkJCAi1atKB169bEx8fj7e1N3759G3p4ImIACvANaMKECZSXlzNr1ixKSkoICQlh5cqVlRZeRUT+DL10W0TEoJSDFxExKAV4ERGDUoAXETEoBXgREYNSgBcRMSgFeBERg1KAFxExKAV4ERGDUoAXETGo/wcdAjTkZlx1GgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "best_alpha=0.05\n",
    "testBernouli(train_tfidf,train_y,test_tfidf,test_y,best_alpha) #Tf-idf testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tPositive\t\t\t\t\tNegative\n",
      "________________________________________________________________________________________________\n",
      "\t-19.6946\taachen         \t\t\t\t-1.1889\tlikeabl        \n",
      "\t-19.6946\taarrgh         \t\t\t\t-1.2055\ttasteand       \n",
      "\t-19.6946\tabbey          \t\t\t\t-1.2798\tgreater        \n",
      "\t-19.6946\tabbrevi        \t\t\t\t-1.2799\tlovee          \n",
      "\t-19.6946\tabdomen        \t\t\t\t-1.2854\tgoodad         \n",
      "\t-19.6946\tabhor          \t\t\t\t-1.4373\tflavorc        \n",
      "\t-19.6946\tabrevi         \t\t\t\t-1.4592\tonec           \n",
      "\t-19.6946\tabsinth        \t\t\t\t-1.4933\tusei           \n",
      "\t-19.6946\tabsolutelt     \t\t\t\t-1.5377\ttriangl        \n",
      "\t-19.6946\tabstin         \t\t\t\t-1.5654\tproducto       \n",
      "\t-19.6946\tacceptal       \t\t\t\t-1.6431\tmakeov         \n",
      "\t-19.6946\taccourd        \t\t\t\t-1.6864\tgetaway        \n",
      "\t-19.6946\taccpet         \t\t\t\t-1.9116\tbuy            \n",
      "\t-19.6946\tacct           \t\t\t\t-1.9129\ttimefram       \n",
      "\t-19.6946\tacesuflamek    \t\t\t\t-1.9500\tamazon         \n",
      "\t-19.6946\tack            \t\t\t\t-1.9674\tfindabl        \n",
      "\t-19.6946\tacknowleg      \t\t\t\t-1.9694\twouldst        \n",
      "\t-19.6946\tacquisit       \t\t\t\t-1.9722\tbest           \n",
      "\t-19.6946\tactualla       \t\t\t\t-2.0033\trealllli       \n",
      "\t-19.6946\taddicitv       \t\t\t\t-2.0259\tpricei         \n"
     ]
    }
   ],
   "source": [
    "def show_most_informative_features(vectorizer, clf, n=20):\n",
    "    feature_names = uni_gram.get_feature_names()\n",
    "    \n",
    "    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))\n",
    "    top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])\n",
    "    print(\"\\t\\t\\tPositive\\t\\t\\t\\t\\tNegative\")\n",
    "    print(\"________________________________________________________________________________________________\")\n",
    "    for (coef_1, fn_1), (coef_2, fn_2) in top:\n",
    "        print(\"\\t%.4f\\t%-15s\\t\\t\\t\\t%.4f\\t%-15s\" % (coef_1, fn_1, coef_2, fn_2))\n",
    "        \n",
    "show_most_informative_features(train_tfidf,bnb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mulnb(train,train_y,cv,cv_y):\n",
    "    best_alpha=0;max_roc_auc=-1\n",
    "    alpha=[100,50,10,5,1,0.5,0.1,0.05,0.01,0.005,0.001,0.0005,0.0001] \n",
    "    for i in alpha[-10:]:\n",
    "        mulbnb = MultinomialNB(alpha=i)\n",
    "        mulbnb.fit(train,train_y)\n",
    "        pred = mulbnb.predict(cv)\n",
    "        #acc = accuracy_score(cv_y, pred, normalize=True) * float(100)\n",
    "        roc_auc=(roc_auc_score(cv_y,pred)*100)\n",
    "        if(max_roc_auc<roc_auc):\n",
    "            max_roc_auc=roc_auc\n",
    "            best_alpha=i\n",
    "    print(f\"\\n Best alpha Value {best_alpha} with highest roc_auc Score is {roc_auc}\")\n",
    "    sns.set_style(\"darkgrid\")\n",
    "    plt.plot(alpha,error_list)\n",
    "    plt.title(r'roc_auc v/s $\\alpha$')\n",
    "    plt.xlabel(r\"alpha values\",fontsize=12)\n",
    "    plt.ylabel(\"roc_auc\",fontsize=12)\n",
    "    plt.show()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "def testmulnb(train,train_y,test,test_y,best_alpha):\n",
    "    bnb = MultinomialNB(best_alpha,fit_prior=True, class_prior=None)\n",
    "    bnb.fit(train,train_y)\n",
    "    pred = bnb.predict(test)\n",
    "    roc_auc=(roc_auc_score(test_y,pred)*100)\n",
    "    print(f\"best roc auc score is {roc_auc}\")\n",
    "    print(\"Confusion Matrix of test set:\\n [ [TN FN]\\n [FP TP] ]\\n\")\n",
    "    df_cm = confusion_matrix(test_y,pred)\n",
    "    sns.set(font_scale=1.4) #for label size                     \n",
    "    sns.heatmap(df_cm, annot=True,annot_kws={\"size\": 16}, fmt='g')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Best alpha Value 5 with highest roc_auc Score is 49.86357275635475\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEgCAYAAABFO1+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XlclOXeBvBrhtkYZoTBWE5hLiQKCIgmiqmhBKXI6Zjl0qsetyOh5jlqr6ZtdkLL40sWlqYpttgprTyox6VyK3fFJdFQTEXFEAhRQJaBmef9A5kYAWcGGAZmru/nwyfnWWZ+N+lcc9/3M88tEgRBABER0X2IbV0AERG1fAwLIiIyiWFBREQmMSyIiMgkhgUREZnEsCAiIpMYFkREZBLDgoiITGJYELVif//73/Huu+/augxyAAwLolaqsrISBw8exIABA2xdCjkAhgVRK3Xy5EmIRCKEhoY2+rm+++47DB8+HMHBwYiKisL3338PABg7diwWLFjQ6Oen1o9hQS3SoEGD8M4772DSpEkICQnBP/7xD+h0Ovz73/9GbGwsQkJC8Pjjj2Px4sUoKysznCcIAj7//HPExMQgODgYkZGR+PDDD6HT6Sx6/bKyMiQmJiI6OhrdunVDjx49MGHCBPzyyy+16nzppZeMth08eBBdunTBkSNHGlTT66+/jj59+qCystJo+wcffICQkBAUFxcDAPbu3Yt+/frByckJAJCeno6JEyeiV69e6N69O55//nkcOHDAZFuXLVuGGTNmoEOHDpg7dy6USiXmzZuHo0eP4sSJE5g8ebJ5vzSyawwLarHWrVsHHx8fLFu2DKNGjcLrr7+OhIQEDBgwAElJSRg9ejS++OILvPDCC6i+H+bSpUuxcOFChIeHY9myZRg5ciRWrFiBpUuXWvTac+fOxYYNGzBx4kR8/PHHmDNnDjIyMjBz5kxYeu9NS2t6+umnUVBQUOuNfuvWrYiMjIRKpQIA/PTTT4YhqOLiYkyYMAFyuRxLlizBe++9B6lUiri4OFy7dq3e2k6cOIEPPvgAY8eORWJiIv7nf/4H8+bNQ3FxMd566y0MHToUPj4+FrWX7JPE1gUQ1Uej0eC1116DRCLBr7/+im+++QYvvvgipk+fDgB4/PHH4eXlhZdffhm7d+9GWFgYkpOTMWrUKLz66quGY+7cuYPDhw9Dr9dDLDb9+Uir1aKwsBDz58/H008/DQAIDw9HSUkJFi9ejJycHHh7e5vVhqKiIotr6tmzJx5++GFs3boVjz/+OADg7NmzuHTpEl5++WUAQHZ2Nn799VdDWFy8eBEFBQX429/+hh49egAAgoODsXLlSqOe170+/fRTuLi44O9//7th20MPPQQA+PXXX/Hee++Z1U6yfwwLarF8fX0hkVT9FT169CgAIDY21uiY2NhYvPLKKzh69ChkMhkqKirw5JNPGh0zc+ZMi15XJpNh7dq1AIDff/8dmZmZuHz5Mvbu3QugKkzMderUqQbV9PTTTyM5ORllZWVQKBTYsmULPDw80K9fPwBVQ1BBQUFwd3cHAHTu3BkeHh6Ij4/HkCFD0K9fP/Tu3Rvz5s2r9zV0Oh3279+P/v37Q61WG7aLRCIAQFRUFHx9fc1uK9k3DkNRi/XAAw8Y/nz79u1a2wBAIpFAo9GgsLAQBQUFdR7TEAcPHkRsbCwee+wxxMXFISUlBTKZDAAsGoZqaE1PP/00SkpKsGfPHuj1emzbtg1Dhw41zE/8+OOPRldBKZVKfPnll4iKisJ3332HqVOnIjw8HLNnz0ZRUVGdr3H16lUUFxcjMDCwzprHjBljUc1k3xgW1Cq4uroCqPqkX1NFRQUKCgqg0WjQpk0bAMDNmzeNjsnNzcWhQ4fuOxxT09WrV/HCCy+gc+fO2LlzJ1JTU/HFF19g4MCBdR6v1+uNHldPQANocE3t2rVDz549sX37dhw9ehQ5OTmGITGtVosjR44gIiKi1jkJCQk4cOAAUlJSMH78eGzfvr3euZFbt24B+ON3W23lypUAqoYBiaoxLKhVCAsLAwBs2bLFaPvWrVuh0+nQs2dPBAcHQyqV4ocffjA65osvvkB8fLzZPYIzZ86gvLwcEydORLt27QzDMj/99BMA43BQqVT47bffjM6vHjID0Kia/vKXv2Dfvn3YtGkT/Pz84O/vDwA4cuQIlEqlUY9gx44d6NOnD3JzcyESieDv74/Zs2ejU6dOuH79ep3PXx1kGRkZhm0//vijoVZzw5UcA+csqFV45JFHMGzYMCxfvhxlZWUICwvDuXPnsHz5cvTq1QsRERFwcnLCuHHjsHbtWkilUvTt2xfp6elYs2YN4uLi4OzsbNZrBQYGQiKRIDExEePHj0dFRQU2btyIH3/8EQBQWlpqOHbQoEFYsWIFkpKS8Oijj+Lw4cPYsWOHYb+7u3uDaxo8eDASEhKwadMmzJ4927C9egiqOsQAoEePHhAEAS+88AImT54MV1dX7Nu3DxcuXKj30tdOnTqhQ4cO+Oqrr6BSqaBQKLBq1So89dRT2LFjB1auXIkpU6YgODjYrN8b2TmBqAUaOHCgMHv2bKNtlZWVwvLly4XIyEghMDBQGDhwoLBkyRKhtLTUcIxerxeSk5OFqKgoITAwUIiOjhbWrl0r6PV6i15/+/btQkxMjBAUFCT069dPmDZtmnD06FGhS5cuwqeffmo4rqSkRHjjjTeEsLAwoXv37sILL7wgHD9+XPDz8xMOHz7c6Jpmzpwp+Pv7Czk5OYZtUVFRwrZt22ode+bMGWHy5MlC7969hW7dugmxsbHCN998c9/nT09PF0aOHCl069ZN6Nmzp7Bw4UKhsrJSWLRokdCtWzdhxYoV5v7KyM6JBMHCi8aJiMjhcBiKHIZer0dFRYXJ45ycnAyX7BJRFf6LIIdx7NgxjBs3zuRx06dPx4svvtgMFRG1HhyGIodRXFyMy5cvmzzO09MTXl5ezVARUevBsCAiIpP4PQsiIjLJLuYsCgruQK+3vIPUtq0K+fnFpg+0I2yzY2Cb7V9j2isWi6DRuFh0jl2EhV4vNCgsqs91NGyzY2Cb7V9ztpfDUEREZBLDgoiITGJYEBGRSQwLIiIyiWFBREQmMSyIiMgkhw2LrYcy8ey8/9q6DCKiVsFhw+LbHy+hXKtDUYnW1qUQEbV4DhsWSnnV9xGLS03fspqIyNE5bFgo5E4AgDullTauhIio5XPYsJBLq8KiuIw9CyIiUxw2LJzvDkPd4TAUEZFJDhsWCtndYagyDkMREZnisGFRPQxVwmEoIiKTHDYsqnGCm4jINKutZ3HkyBGMGzeuzn0+Pj7YtWsXEhMTsWrVqlr7z549C4mkeZbauMOeBRGRSVZ7Rw4NDcX+/fuNtmVkZGDKlCmIi4sDAJw/fx4jRozAjBkzjItqhqCoXnmcV0MREZlmtXdlmUwGDw8Pw+OKigosWrQIUVFRGDFiBICq8Bg4cKDRcc2Nw1BERKY127Kqn3/+ObKzs5GcnAwAKCwsRHZ2Nh555JHmKqFOHIYiIjKtWSa4S0tLsXLlSowbNw5eXl4AqnoVALBlyxZER0dj4MCBmDt3LnJzc5ujJIMSXjpLRGRSs/QsNm3ahPLycqMJ7+qwUKlUSEpKQl5eHpYuXYqxY8ciJSUFzs7OZj9/27Yqi2uSyv64dLZtWxXEYpHFz9FaeXiobV1Cs2ObHYOjtbk529tsYREVFQV3d3fDttGjRyMmJgaurq4AgK5du8LPzw+PP/44du7cidjYWLOfPz+/GHq9YFFN2vKqHoVeAK5dL4BSIbXo/NbKw0ONvLwiW5fRrNhmx+BobW5Me8VikcUfsq0+DHXz5k2cOnUKQ4cONdouEokMQVHNy8sLbm5uyM7OtnZZRoo5FEVEdF9WD4sTJ05AJBKhV69eRtsTEhLwl7/8xWjbtWvXUFBQ0CyT3jX7Ibw/FBHR/Vk9LH755Re0a9cOSqXSaPtTTz2FCxcuICEhAZmZmTh69CimT5+O4OBgREREWLssI7wiiojo/qweFnl5ebWGmwDg0UcfxUcffYS0tDQMGzYML774IgIDA7Fq1SqIxc1zFxKZpOp1eEUUEdH9WX2C+6233qp3X//+/dG/f39rl1AvpbMU2qJylGl1NquBiKg1cOgbCbooqrKytJw9CyKi+3HosFDIGRZEROZw6LAQiURQyJxQWs5hKCKi+3HosACqllct1bJnQUR0PwwLuQRlHIYiIrovhoXMiXMWREQmOHxYKOQSlPLSWSKi+3L4sGDPgojItGZb/KilUcicAJGoas6CPQsiovty2LB4/gk/uGmU+Pf2dJSwZ0FEdF8OOwzVxkWGtq7OUMicUK7VWbweBhGRI3HYsKjmfPdb3ByKIiKqH8PCEBYciiIiqg/D4m5YcN6CiKh+DAuZEwCgjPeHIiKql8OHheHOsxyGIiKql8OHhTNvU05EZBLD4u4wFMOCiKh+DAtDz4JzFkRE9XH4sJBXT3BzzoKIqF4OHxZirpZHRGSSw4cFcHe1PM5ZEBHVi2EBLq1KRGQKwwJVV0RxaVUiovpZ7RblR44cwbhx4+rc5+Pjg127diErKwtvvfUWjh07BoVCgWHDhmHmzJmQSJr3zukKDkMREd2X1d6VQ0NDsX//fqNtGRkZmDJlCuLi4qDVajFp0iR07NgRX331Fa5du4b58+dDIpFg5syZ1iqrTkq5BL/fLmvW1yQiak2sNgwlk8ng4eFh+HFzc8OiRYsQFRWFESNG4LvvvsP169exePFi+Pn5ITIyEi+99BI+++wzlJU17xu3q0qGW0XlEASuaUFEVJdmm7P4/PPPkZ2djXnz5gEAUlNT4e/vD1dXV8MxvXv3RklJCc6ePdtcZQEA3NUKlFfoePksEVE9miUsSktLsXLlSowbNw5eXl4AgJycHHh7exsd5+npCQC4ceNGc5Rl4KaWAQAKisub9XWJiFqLZplJ3rRpE8rLy40mvMvKyuDi4mJ0nExW9aZdXm7Zm3bbtqoG1+bhoUandloAgF4shoeHusHP1Vo4QhvvxTY7Bkdrc3O2t9nCIioqCu7u7oZtCoUCWq3W6Ljqx0ql0qLnz88vbtAa2h4eauTlFQG6quGnzKwCtHN3tvh5WhNDmx0I2+wYHK3NjWmvWCyy+EO21Yehbt68iVOnTmHo0KFG2729vZGbm2u0rfrxvcNT1qZRVfVobhVxGIqIqC5WD4sTJ05AJBKhV69eRtt79eqF9PR0FBYWGrYdOXIELi4uCAgIsHZZRqQSJ6icpShgWBAR1cnqYfHLL7+gXbt2tYaWnnjiCXh5eWHmzJk4d+4cdu/ejcTEREyYMMEwd9Gc3NVyhgURUT2sHhZ5eXlGl8dWk8vlWL16NQBgxIgReP311zFy5EhMmzbN2iXVyY1hQURUL6tPcL/11lv17mvfvj3WrFlj7RLMolHLcTm70PSBREQOiDcSvEujlqOopAIVlXpbl0JE1OIwLO7SqOQAgFv8Yh4RUS0Mi7s0barCgvMWRES1MSzuqu5ZMCyIiGpjWNylUSsAMCyIiOrCsLjLWe4EudSJYUFEVAeGxV0ikQgatZx3niUiqgPDogaNWo6CIq6YR0R0L4ZFDRq1nDcTJCKqA8OiBo1ajlvFWui5vCoRkRGGRQ0atRw6vYCiO1rTBxMRORCGRQ3V37W4yaEoIiIjDIsa3NR3b/nBsCAiMsKwqMFdzZ4FEVFdGBY1qF1kcBKLeDNBIqJ7MCxqEItEcFPJcLOQYUFEVBPD4h5uajl7FkRE92BY3EOjVnDOgojoHgyLe2hUVd/iFvjFPCIiA4bFPTRqOcordCgt19m6FCKiFoNhcQ+NunoRJN5QkIioGsPiHoaw4CQ3EZEBw+IehrDg5bNERAYWhcWxY8cAALdu3cLOnTvNOiclJQVDhgxBUFAQYmJisH37dsO+xMREdOnSpdZPZWWlJWU1KTcVexZERPeSmHvg0qVLceLECXz++ecoKyvDqlWrkJGRgalTp9Z7zqZNmzB//nzMnTsXERER2LZtG2bNmgVPT0/07NkT58+fx4gRIzBjxgzjoiRml9XkpBIx1Eopl1clIqrB7J7Frl27kJycDADw9vbGunXrsG3btnqPFwQB77//PsaMGYO//vWvaN++PeLj49G3b18cPnwYAJCRkYGAgAB4eHgY/diaRiVnWBAR1WD2R/iKigpIpVLDY6lUCpFIVO/xly5dwvXr1zF06FCj7WvWrAEAFBYWIjs7G4888oilNVudRi3nF/OIiGowu2fRo0cPzJ49G4cOHcLhw4cxb948hISE1Ht8ZmYmAECr1WLKlCkIDw/Hc889h927dwOo6lUAwJYtWxAdHY2BAwdi7ty5yM3NbURzmkbVWtwMCyKiaiLBzK8ql5SU4P3338ehQ4cgkUgQHh6O6dOnw9nZuc7jN23ahDlz5uChhx7CtGnTEBAQgO+//x4rVqzAmjVrcOXKFbz55puYNGkS/vznPyMvLw9Lly7FnTt3kJKSUu/zNof1P5zHuh3n8O07QyGTOtmsDiKilsLsYSilUol58+aZ/cTVQ1YTJkzA8OHDAQD+/v44c+YMkpOTsXr1asTExMDV1RUA0LVrV/j5+eHxxx/Hzp07ERsba/Zr5ecXQ6+3/PYcHh5q5OUV1a79bn/rQmY+PN1sF1rWUF+b7Rnb7Bgcrc2Naa9YLELbtiqLzjE7LF544YU6t3/00Ud1bvf29gYA+Pn5GW3v3Lkzdu3aBZFIZAiKal5eXnBzc0N2dra5ZVmFu1oBoGrFPHsLCyKihjA7LJ588knDnysqKvDdd9+hW7du9R4fEBAAFxcXpKWloXfv3obtGRkZePjhh5GQkIDU1FSkpKQY9l27dg0FBQU2n/R2M6yYx1t+EBEBFoTFsGHDaj0eO3ZsvccrFApMnjwZy5cvh6enJ7p3746tW7di//79WLt2LaRSKb788kskJCRgzJgxyM3NxcKFCxEcHIyIiIgGN6gpuBvW4tbatA4iopaiwd9+EwTB5JVLU6dOhVKpRFJSEm7cuIFOnTph2bJlCA8PB1A1hPXBBx9g2LBhkMlkiIyMxP/+7/9CLLbtXUic5RLIZU7sWRAR3dXgOYuMjAyEhYWZPG/8+PEYP358nfv69++P/v37m1tCs3JXV61rQUREDZyzEIlEGD16NPr162eVoloCN36Lm4jIoMFzFoIg4MqVK+jQoUNT19QiaNRynLtaYOsyiIhaBLPD4quvvsK//vUvlJaWGra5u7vjwIEDVinM1jRqOW4VaaHXCxCL67+tCRGRIzA7LFatWoW1a9dixYoV+Mc//oE9e/bgxo0b1qzNpjRqOfSCgMISreG25UREjsrsy47c3NwQEhICf39/5OfnIz4+3rC+hT36Y3lVzlsQEZkdFhKJBLdv30b79u1x+vRpAIBOp7NaYbbGsCAi+oPZYTFixAjExcUhIiIC69evxzPPPANfX19r1mZTmru3/GBYEBFZMGfx7LPPYsiQIVAqlVi/fj3S0tIMl87+97//rbVuRWunVkrhJBYxLIiIYOEa3EqlEkDVDf+eeOIJKBRVn76rFzSyJ2KRiN+1ICK6q0nuq2HmkhitTtUiSLzlBxFRk4TF/ZZXbc00ajkKinkzQSIi296xr4Wr7lnYa8+JiMhcDIv70Kjl0FboUVpeaetSiIhsinMW96ExLILESW4icmwWhUX1N7Zv3bqFnTt3GrZbsl52a1J9mw/eqpyIHJ3ZYbF06VIkJSUBAMrKyrBq1SosX74cADBp0iTrVGdj7uxZEBEBsCAsdu3aheTkZACAt7c31q1bh23btlmtsJbATc2eBRERYEFYVFRUQCqVGh5LpVK7vWS2msRJjDZKKXsWROTwzL7dR48ePTB79mw8++yzEIlESElJQUhIiDVraxHc1HLcKmZYEJFjM7tn8dprr+GBBx7A22+/jX/9619o27YtXnnlFWvW1iK4qxW85QcROTyzexZKpRLz5s1DVlYWdDod2rdvb826Wgw3tRy/Xr9t6zKIiGzK7LDIzMzEtGnTkJubC0EQ4ObmhpUrV9r1bcqBqu9aFJdWoKJSB6nEydblEBHZhNnDUG+99RYmT56MY8eOITU1FfHx8XjzzTetWVuLoFFxESQiIrPDIj8/H8OGDTM8Hj58OAoKCqxSVEuiacOwICIyOyx0Oh1u3bpleHzz5k2zzktJScGQIUMQFBSEmJgYbN++3bAvKysLcXFx6NGjB/r27YslS5agsrJl3YeJPQsiIgvmLMaMGYORI0di8ODBEIlE2LZtG/7617/e95xNmzZh/vz5mDt3LiIiIrBt2zbMmjULnp6eCAoKwqRJk9CxY0d89dVXuHbtGubPnw+JRIKZM2c2umFNxbAWNy+fJSIHZnZY/Pvf/8abb76Jffv2Qa/X44033kDfvn3rPV4QBLz//vsYM2aMIVTi4+ORmpqKw4cP47fffsP169exYcMGuLq6ws/PDy+99BIWLVqE+Ph4wyp8tuYsl0Ahc0JBIcOCiByX2WHh7OyMDh06oE+fPmYdf+nSJVy/fr3W2tzVS7C+8cYb8Pf3h6urq2Ff7969UVJSgrNnz6Jnz57mlmZ1VYsgMSyIyHGZHRalpaWIjIyEt7e3YS1uANiyZUudx2dmZgIAtFotpkyZgrS0NPj4+CA+Ph6DBg1CTk4OvL29jc7x9PQEANy4ccOiRrRtq7Lo+Jo8PNQmj/Fyd0FxaaVZx7YG9tIOS7DNjsHR2tyc7TU7LCz9tnZxcTEAYM6cOZg2bRpmzpyJ77//HlOnTsWaNWtQVlYGFxcXo3NkMhkAoLzcsk/x+fnF0OstX1PDw0ONvLwik8e5yJ1w5UahWce2dOa22Z6wzY7B0drcmPaKxSKLP2SbHRZhYWEWPXH1TQcnTJiA4cOHAwD8/f1x5swZJCcnQ6FQQKs1Xt+6+nHNnktLoGkjx+1iLfR6AWKxfd88kYioLlZbVrV6iMnPz89oe+fOnZGVlQVvb2/k5uYa7at+fO/wlK1pVHLoBQG372hNH0xEZIesFhYBAQFwcXFBWlqa0faMjAw8/PDD6NWrF9LT01FYWGjYd+TIEbi4uCAgIMBaZTWIRl11ZRbvPktEjspqYaFQKDB58mQsX74cmzdvxtWrV7FixQrs378fEydOxBNPPAEvLy/MnDkT586dw+7du5GYmIgJEyYY5i5aCsNa3Lx8logclNlzFg0xdepUKJVKJCUl4caNG+jUqROWLVuG8PBwAMDq1avxz3/+EyNGjECbNm0wcuRITJs2zZolNUh1WLBnQUSOyqphAQDjx4/H+PHj69zXvn17w/cuWjKVUgonsQg3i8psXQoRkU1YbRjKnohFImjUcq7FTUQOi2FhJje1nDcTJCKHxbAwkzvDgogcGMPCTG6qqrAQBMu/KU5E1NoxLMykUcuhrdSjpLxlrbdBRNQcGBZmMqxrwe9aEJEDYliYiYsgEZEjY1iYyRAWnOQmIgfEsDCTG9fiJiIHxrAwk8RJjDYuMoYFETkkhoUFNCp+14KIHBPDwgIafjGPiBwUw8ICVWHBmwkSkeNhWFhAo5bjTlkltBU6W5dCRNSsGBYW4HctiMhRMSwsYFgEifMWRORgGBYWMCyvyrAgIgfDsLBA9Rfz2LMgIkfDsLCAs1wCZ7kTexZE5HAYFhZyU3F5VSJyPAwLC7mr5bwaiogcDsPCQlyLm4gcEcPCQhq1AreLtdDp9bYuhYio2Uis+eSXLl3C4MGDa21PSEjAc889h1mzZmHr1q1G+7y8vPDTTz9Zs6xG0ajl0AsCCu9UGC6lJSKyd1YNi/Pnz0OlUmHHjh1G29VqtWH/jBkzMGLECMM+Jycna5bUaDUXQWJYEJGjsGpYZGRkwNfXFx4eHrX2abVaZGZmIigoqM79LZXGsAhSGYA2ti2GiKiZWHXO4vz58/D19a1z38WLF1FZWYlHHnnEmiU0OU0brphHRI7HqmGRkZGB3NxcjBo1Cn379sXzzz+P/fv3A6gKEolEgpUrV2LQoEGIiopCQkICioqKrFlSo6mdpZA4iRgWRORQrDYMVVJSgqysLLi7u2P27NlwcXHB5s2bMXnyZCQnJ+PChQsAAB8fH3z00Ue4cuUKFi9ejHPnzuGzzz6DWGx+jrVtq2pwnR4eaovPaevqjNIKfYPObQlaa92NwTY7Bkdrc3O2VyQIgmCtJ79z5w6kUilkMplh26RJkyAIAlavXo3i4mK0afPHuP/JkycxatQofPnll+jRo4fZr5OfXwy93vJmeHiokZdneU/m7XXHIRaJMPd/zK+xpWhom1szttkxOFqbG9NesVhk8Ydsqw5Dubi4GAUFAPj5+eG3336DWCw2CgoA6NKlCwAgOzvbmmU1mobf4iYiB2O1sDh58iRCQ0Nx+vRpo+1nzpxB586dMW3aNMTHxxvtqz62pU96V6/F3ZDeDBFRa2S1sOjWrRt8fHzw2muv4fjx47h48SISEhJw8uRJxMfHY8iQIdi9ezdWrVqFq1evYu/evZg/fz6efPJJQw+jpfLzcUNFpR5H0nNsXQoRUbOw2gS3VCrF6tWrkZiYiBkzZqCwsBCBgYFITk5GQEAAAgICoNfrsXr1anz44YdQq9WIiYnBzJkzrVVSkwnp/AB8PFyw5UAmevt7QSwW2bokIiKrsuoEd3Np7gluAEg9l4vlKWcwJTYAfQK9G/QctuBok4AA2+woHK3NdjXBbc96dPHAQx4u2HIwk3MXRGT3GBYNJBaJ8OfHOiI7vwTHzuXauhwiIqtiWDRCzy4eeOgBF2w+cJm9CyKyawyLRhCLRIh9rAOy80uQep69CyKyXwyLRnq0qycefMAFmw9kQt/6rxUgIqoTw6KRxCIRYvt2wG+/30Eq5y6IyE4xLJpAr66e+FNbJbawd0FEdoph0QTE4qq5i+u/38GJ83m2LoeIqMkxLJpIWFcv/KmtsurKKPYuiMjOMCyaiFhcNXeRlcfeBRHZH4ZFEwrz94K3u5JXRhGR3WFYNKE/ehfFOJnxu63LISJqMgyLJhYW4AkvjTPnLojIrjAsmpiTWIzYxzrgWm4xTl1g74KI7APDwgp6B3jB827vwg7uAE9ExLBUL2EgAAASA0lEQVSwBiexGLF9O+BqTjFO/creBRG1fgwLK+kT6AVPN2ds2s/eBRG1fgwLK3ESizH0bu/i51/zbV0OEVGjMCysKLybFzzcFNjEuQsiauUYFlZU3bu4cqMIP19k74KIWi+GhZWFB3rjAVcFNnPugohaMYaFlUmcqnoXmTeKkHaJvQsiap0YFs2gb7eq3sWm/ZnsXRBRq2TVsLh06RK6dOlS6+frr78GAKSnp2Ps2LHo3r07IiIisGbNGmuWYzPVvYvL2YVIu3TT1uUQEVlMYs0nP3/+PFQqFXbs2GG0Xa1W4+bNmxg/fjyioqKwYMECnD59GgsWLIBarcaIESOsWZZN9O3mjS0HMrH5wGUEdXKHSCSydUlERGazalhkZGTA19cXHh4etfZ98sknkEqlWLBgASQSCXx9fXHlyhWsWrXKLsNC4iRGTN/2+GzHeZy9fBPdOrW1dUlERGaz6jDU+fPn4evrW+e+1NRUPProo5BI/sir3r1749q1a8jJybFmWTbTL+hPaNtGjpT9l1FQVI7yCh3nMIioVbB6z6J9+/YYNWoUrl69ig4dOmDq1Kno168fcnJy8Mgjjxgd7+npCQDIzs6Gl5eXNUuziareRQd8tuM8Zn94AADgJBbBRSGBs0IKF4UESrkESoUEyhqPnRUSuCikNfZVPXaWO8FJzGsUiMj6rBYWJSUlyMrKgru7O2bPng0XFxds3rwZkydPRnJyMsrKyiCTyYzOqX5cXl5u0Wu1batqcJ0eHuoGn9sQzz7RBR19NMi7VYriEi3ulFaguLTC6L/5eXcMj/X6+/c8nOUSuDhLoXKWGv9XKYVKIYWL8u42hRQqpQx3KgsNxyhkTg4zd9Lc/59bArbZ/jVne60WFkqlEsePH4dUKjWEQLdu3XDx4kWsXr0aCoUCWq3W6Jzqx0ql0qLXys8vNvmmWhcPDzXy8oosPq+x2j+gRPsHTLdREASUV+hQUlZZ9VNeiTtlFbUel5ZV4s7dx9eLyu5ur0S5Vnff53cSi6rCRvFHb0Z593F9vZnqno5SLoHEqXX0amz1/9mW2Gb715j2isUiiz9kW3UYysXFpdY2Pz8/7NmzB+3atUNubq7RvurH3t7e1iyr1RCJRFDIJFDIJHBvY/n5Or0epeU6o4CRyCXIzi0yCpiSu/vvlFXi99tlhsc6EwEslzr9ESTyu2FTz1BazcfOcolD9WqI7IHVwuLkyZOYOHEiPv30UwQHBxu2nzlzBp07d0ZQUBC++OILVFZWGia5Dx8+jA4dOtR59RRZzkkshspZDJWz1LDN3E8jgiBAW6m/GzIVtYKlrp5OfmEZruVWoKS8EqXl9+/ViEWie4LFuGdjqqfTWno1RPbCamHRrVs3+Pj44LXXXsPrr78ONzc3fPnllzh58iQ2bNgALy8vrF69GvPnz8eUKVNw5swZfPLJJ3jjjTesVRJZQCQSQS51glzqBI1abvH5er1QFS4mAqbmUNrNwnLD8ZW6+/dqZFKx0bCYi7z+gHnwdjm0ZVrDUBp7NUSWEwlWvHYzJycHiYmJOHDgAAoLCxEYGIhZs2YhLCwMAJCWloaFCxfi7Nmz8PDwwPjx4zFu3DiLX6e1zVnYUmtps7ZCZ5h7qRoyq7gbJPf2dO6GkeHPlSgtr8T9/jaIRDAaGjPqzcilNXo6VY/v7elIJS2/V9Na/j83JUdrc3PPWVg1LJoLw8J8jtBmvV5AmfbunExZJaQKKbJzCg2PS8qNezYl94RRRaX+vs8vk4jrvgBALjVM/rsoJJDbsAfTpo0ChYVlNnltW3G0NnfwcYOHSmb6wDq0uAluIlsQi0V3J9ur5mo8PNR40E1h9vkVlboaQ2TGvZe6ejq3i7XIzr9jCKBW/+mLWgWJkwjL/j4AcplT87xes7wKUSsilTjBVeUEV1UD5moEAWXlOpSUV5i8dNmaNO4uKLh5x2avbwuO1ub2PhpUllc02+sxLIiakOEqL4Vt/2l5eKihdHKsSXxHa7OmjQJ5ec0XFi1/po6IiGyOYUFERCYxLIiIyCSGBRERmcSwICIikxgWRERkkl1cOisWN/xyucac21qxzY6BbbZ/DW1vQ86zi9t9EBGRdXEYioiITGJYEBGRSQwLIiIyiWFBREQmMSyIiMgkhgUREZnEsCAiIpMYFkREZBLDgoiITHK4sNDr9UhKSkL//v0REhKCiRMn4sqVK7Yuq0kVFxdj0aJFGDRoEEJDQ/HMM89g165dhv1ZWVmIi4tDjx490LdvXyxZsgSVlZU2rLjpXL58GaGhofj6668N2+y5vSkpKRgyZAiCgoIQExOD7du3G/bZY7u1Wi0SExMRERGB0NBQPP/88zhx4oRhvz21eeXKlRg9erTRNnPa98UXXyAyMhLBwcEYOXIkTp8+3TQFCQ4mKSlJ6NOnj7B3714hPT1dmDx5shAZGSmUlZXZurQmM336dCEqKko4cOCAkJmZKaxYsULo2rWrcPDgQaG8vFyIjo4W4uLihPPnzws7d+4UwsLChHfffdfWZTeaVqsVnnnmGcHPz0/YsGGDIAiCXbc3JSVF8Pf3Fz755BMhMzNTWL58udC1a1chNTXVbtv97rvvCuHh4cJPP/0kXL58WXjllVeE0NBQ4caNG3bV5nXr1gldunQRRo0aZdhmTvu+/fZbISQkRNi8ebNw4cIFYe7cuUKvXr2E33//vdE1OVRYlJeXC927dxfWrVtn2FZUVCSEhIQI//nPf2xYWdPJzc0V/Pz8hD179hhtHzdunDBr1ixh8+bNQmBgoHDr1i3Dvg0bNgjdu3cXSktLm7nappWYmCiMHTvWKCzstb16vV4YOHCgsHDhQqPtEydOFD744AO7bXdMTIzw9ttvGx4XFRUJfn5+wtatW+2izTdu3BDi4uKE7t27C0899ZRRWJjTvujoaGHx4sWG/ZWVlcLAgQOFDz74oNG1OdQwVHp6OkpKStCnTx/DNpVKhYCAAKSmptqwsqbj7OyMjz/+GI8++qjRdpFIhNu3byM1NRX+/v5wdXU17OvduzdKSkpw9uzZ5i63yRw7dgzr16/H4sWLjbbba3svXbqE69evY+jQoUbb16xZg2nTptltu93d3bFnzx5kZWVBp9Ph66+/hkwmM/wbbu1tPnv2LFxcXLB582aEhIQY7TPVvt9//x2ZmZno3bu3Yb+TkxN69uzZJO9vDhUWOTk5AAAvLy+j7Z6ensjOzrZFSU1OpVJhwIABUKlUhm2nTp3C4cOHERERgZycHHh7exud4+npCQC4ceNGs9baVAoLCzFnzhy8+uqr+NOf/mS0zx7bCwCZmZkAqsbwp0yZgvDwcDz33HPYvXs3APtt92uvvQaJRILIyEgEBQUhMTERS5cuRYcOHeyizYMGDUJiYiLatWtXa5+p9lW/v9V1TFO8vzlUWJSWlgIAZDKZ0XaZTAatVmuLkqzu4sWLmD59OkJCQjBy5EiUlZXV2X4AKC8vt0WJjbZgwQJ0794dsbGxtfbZY3uBqosYAGDOnDl48sknkZycjH79+mHq1Kk4cOCA3bb7woULUKlU+PDDD7F+/Xo888wzmDNnDs6ePWu3ba5mqn3Wfn+zi8WPzKVQKABUfRqr+QvVarVQKpW2Kstqjh07hunTp+PBBx/EypUrIZVKoVAoav3FqX7cGn8HKSkpSE1NxZYtW+rcb2/trSaVSgEAEyZMwPDhwwEA/v7+OHPmDJKTk+2y3b/99hvmzJmD1atXG4aSg4KC8OuvvyIpKcku21yTqfbVfH+795imaL9D9Syqhyhyc3ONtufm5tYammrtNm/ejAkTJiAwMBCff/453NzcAFR1Uetqf/W+1ubbb79Ffn6+4VLK0NBQAMA///lPxMTE2F17q1XX7ufnZ7S9c+fOyMrKsst2nz59GhUVFQgKCjLaHhISgszMTLtsc02m2vfggw8abat5TFO8vzlUWHTt2hUqlQpHjx41bCsuLsYvv/yCsLAwG1bWtLZs2YI5c+Zg8ODBWLlypdH8Ra9evZCeno7CwkLDtiNHjsDFxQUBAQG2KLdR/u///g/btm1DSkqK4QcApk+fjlWrVtlde6sFBATAxcUFaWlpRtszMjLw8MMP22W7q9/wz58/b7Q9IyMDHTt2tMs212Sqfe7u7ujYsaPR+5tOp8Px48eb5v2t0ddTtTLvvvuuEBYWJvzwww+G71lER0cL5eXlti6tSWRnZwshISHCuHHjhJycHCE3N9fwU1BQIJSVlQlPPPGEMHHiRCE9PV3YtWuX0Lt3byEpKcnWpTeZmpfO2nN7P/zwQyE0NFTYtGmTcOXKFWH58uVCly5dhIMHD9plu3U6nTB69GjhySefFA4dOiRcvnxZWLp0qdC1a1fh+PHjdtfmuXPnGl06a0771q9fLwQFBQnffvutcOHCBeHll18WwsLChPz8/EbX43BhUVlZKSxZskQIDw8XunfvLkyaNEm4evWqrctqMp9++qng5+dX50/1X7zMzExh4sSJQlBQkPDYY48J7777rqDT6WxcedOpGRaCYN/tXbt2rRAZGSkEBgYKsbGxwvfff2/YZ4/tvnXrlrBgwQIhIiJC6N69uzBy5Ejh0KFDhv321OZ7w0IQzGvfmjVrhAEDBgjBwcHC6NGjhTNnzjRJPSJBEITG90+IiMieOdScBRERNQzDgoiITGJYEBGRSQwLIiIyiWFBREQmMSyIiMgkhgU5nJdffhlr1qy57zEbN25EXFxcM1VUJSsry3C7EqKWhmFBREQmOdRdZ8lx6PV6LFq0CD///DPu3LkDQRCQkJCAnj17Gh0XEBCAv/3tb9i3bx9KSkowa9YsREdHAwDy8vIwZcoUZGdnw8nJCYmJifD19cWpU6ewZMkSaLVa5OXloW/fvli0aJHR816+fBmjRo3Cvn37IJPJoNPpEBERgU8++QRFRUUmz1+2bBkKCgrw+uuv13pcVFSEhQsXIiMjAxUVFQgPD8ecOXMgkUiQlJSEH374AVKpFBqNBm+//bZhzQOixmDPguzSzz//jNzcXKxfvx7btm3DsGHD8PHHH9c6TqfTwdnZGRs3bsR7772H+fPn4+bNmwCAa9eu4ZVXXsGWLVvw6KOPGoauPvvsM8yYMQNff/01tm7dit27d+PMmTNGz9uxY0d07tzZsBjR/v374ePjA19fX7POv59FixYhMDAQGzduREpKCgoKCrB27VpkZ2fj008/xbfffouNGzfisccew+nTpxv6KyQywp4F2aXQ0FC4urriq6++wrVr1wx356zLmDFjAFTdldjPzw/Hjh0DAAQHB6N9+/YAqtaK+OGHHwAA77zzDn766Sd89NFHuHTpEsrLy1FSUlLreZ999ln85z//wVNPPYWNGzdixIgR9z2/+jbypuzduxdpaWn45ptvAFQtigNUrQDZtWtXDBs2DAMGDMCAAQMQHh5u7q+M6L4YFmSX9u7di4ULF2LChAmIjIxEp06dsHnz5jqPdXJyMvxZr9cbHkskf/zzEIlEqL6N2pgxY9ClSxf0798fgwcPxs8//4y6brE2ePBgvPPOO7h48SKOHTuGd955x+zza74eAFRUVBjV+P7778PX1xdA1bKyIpEIYrEY69atQ1paGg4dOoRFixahf//+mDNnjkW/O6K6cBiK7NKBAwcwcOBAPP/88+jWrRt27twJnU5X57HVa2CcPXsWly9fRq9evep93sLCQqSlpeGll15CdHQ0bty4gatXr0Kv19c6Vi6XIyYmBi+//DKio6Ph7Oxs9vkajQZnz56FIAgoLi7Gnj17DPv69euHTz75BIIgQKvVIj4+HuvWrcO5c+cwdOhQ+Pr6Ii4uDuPHj6+13gVRQ7FnQXZp1KhRmD17NmJjY1FZWYnHHnsM33//fZ1v6idOnMCGDRug1+uxdOlSuLq61vu8bdq0wZQpUzBs2DAolUp4eXmhR48euHLlSp1DPs899xzWrVuHBQsWmDy/Xbt2hvP+/Oc/Y9++fYiOjoaXlxfCwsIMPY1XXnkFCxcuRGxsLCoqKtC3b19MnjwZUqkUgwcPxvDhww3LbL766quN/E0SVeEtysmhdenSBYcOHYK7u7utSyFq0TgMRUREJrFnQUREJrFnQUREJjEsiIjIJIYFERGZxLAgIiKTGBZERGQSw4KIiEz6fyC3ckML+kJ6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mulnb(train_bow,train_y,cv_bow,cv_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best roc auc score is 50.0\n",
      "Confusion Matrix of test set:\n",
      " [ [TN FN]\n",
      " [FP TP] ]\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEFCAYAAADpIfy5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XtcVVX+//EXFwnheCgyQbMUf47XSVMSynJyxOymfcdUmNQZR7RMgZQyv5WXTLTvGJn3GtKYsHGiNHK0yWrSaSYzL9CURngXSRRQZxC5y+H8/mA8eULhkOLBvd/PHvuPs/Y6a69d+unDZ6+98LDb7XZERMRwPN09ARERaRwK8CIiBqUALyJiUArwIiIGpQAvImJQCvAiIgalAC8iYlAK8CIiBqUALyJiUArwIiIGpQAvImJQCvAiIgbl7daL+9zozstLEzQkuLe7pyBN1Ps5Gy7p+2dPHnK5b7OWHS7pWk2FWwO8iMgVU21z9wyuOAV4ETEHe7W7Z3DFKcCLiDlUK8CLiBiSXRm8iIhB2arcPYMrTgFeRMxBD1lFRAxKJRoREYPSQ1YREWPSQ1YREaNSBi8iYlC2s+6ewRWnAC8i5qASjYiIQalEIyJiUMrgRUQMShm8iIgx2av1kFVExJiUwYuIGJRq8CIiBqXNxkREDEoZvIiIQakGLyJiUPqFHyIiBqUMXkTEmOx2PWQVETEmZfAiIgalVTQiIgalDF5ExKC0ikZExKBUohERMSiVaEREDEoBXkTEoFSiERExKD1kFRExKJVoREQMSiUaERGDMmEG7+nuCYiIXBHV1a4fDVBZWcmCBQvo378/vXr1YuTIkXz11VeO80ePHmXChAn07t2bvn37kpiYSFWV8/OA1atXExERQY8ePYiKimLXrl1O510Z40IU4EXEHOx2148GWL58Oe+99x4JCQm8//77dOjQgfHjx5Ofn09lZSXjxo3Dw8OD1NRUEhISWLt2LUuXLnV8Py0tjcTERKZMmUJaWhohISGMHz+eU6dOAbg0xsUowIuIOVRVuX40wKZNm3jooYfo168f7du355lnnqGkpISMjAw+/vhjcnNzmT9/Pp06dSIiIoKpU6eyatUqysvLAUhKSmLkyJEMGTKEjh07Mm/ePCwWC6mpqQAujXExCvAiYg72atePBggMDOTvf/87R48exWazsWbNGnx8fOjWrRvp6el07dqVgIAAR//w8HBKS0vJzMzk5MmTZGdnEx4e7jjv5eVFaGgo6enpAPWOURc9ZBURc2hAbb2oqIiioqJa7VarFavV6tQ2c+ZMpkyZQkREBF5eXnh6erJo0SLat29Pfn4+wcHBTv1btWoFQF5eHr6+vgAX7LN7926AeseoiwK8iJhDA2rrKSkpLFu2rFZ7bGwscXFxTm379+/HYrGwfPlygoKCWLNmDdOmTeOtt96ivLwcf39/p/4+Pj4AVFRUUFZW5tR2fp/KykqAeseoiwK8iJhDAzL4MWPGMHTo0FrtP87ejx07xrRp01i5ciW33347ALfccgsHDhxgyZIl+Pr6OgL1Oec++/n5OTL4C/Xx8/MDqHeMuijAi4g5NCDAX6gUcyG7du3i7Nmz3HLLLU7tPXv2ZPPmzdxxxx1kZWU5nSsoKABqyjJt2rRxtHXu3NmpT1BQkKNfXWPURQ9ZRcQU7Daby4erzgXYvXv3OrXv27ePkJAQ+vTpQ1ZWllM9f/v27fj7+9OtWzcCAwMJCQlhx44djvM2m42MjAzCwsIA6h2jLgrwImIOjfCiU48ePQgNDeW5555j27ZtZGdns2jRIrZu3cpjjz3GwIEDCQoKIj4+nj179rB582YWLFjA2LFjHXX06OhoUlJSSEtL48CBA8yYMYOSkhJGjBgB4NIYF+NhtzdwVf9l5O1zo7su3WSMix7J1Kcm0bZtMN988x1Tn36Bbdsz3D0ttxkS3NvdU7hsPD09GRw9hHseuZeWN97AidwCNq76kI0pf3X0GR4byaBR92ENtJKV/h0rZ71O7sGjjvP+Af6MeW4soQNuo9k1PmTtyCR5zkryc/J/uI6XJ5GTf82AEQOxBlrJ2XuEt+ansPsL57chr3bv52y4pO+XvhZXf6f/8ptY/0tE55w+fZpFixbx2WefUVhYSOfOnZkyZYqjJn/kyBHmzJnDzp07sVqtDBs2jMmTJ+Pp+UN+nZycTEpKCoWFhXTv3p3p06fTvXt3x3lXxrgQBXg3Gj16OG+seIW58xaSnv4NMZPG0rdvH3rfdg/Z2d+7e3puYaQAHxX/CA9PHM6aJe+w91976BbWnRFxUfzppVWs+0MakVN+zcOThvPW/6VQcDSfEXFRBAZfzxMRkyg9UwrAzFWz6dC9Aykv/pHiwmJ+/eQoLAEWpgyKpby05iWXx+ZOpP/D/Ul58U3yjuRx32/up1f/3jx1/xSn/1lc7S45wC+PdbmvX0ztFTRXIz1kdaPZs6ayYuVqEuYuBOBvn/6T7779J5OfeJT4J2e5eXZyKTw8PHho/K9Yl5TG2mXvArD7i10EBAbwq8eG8tFbG/nVY0N5Z+Hb/PWPNYHrux2ZvL41mYFR97B+5V8IuD6A3v1DWTZ1MZ+993cAjh3KZfk/kujVvzdffriV1u1bc+/o+3h50ny+/HArAJnbdrPwoyX0uLOHoQL8JTPhZmMK8G7SsWMI7dvfxAcffOJoq6qq4sONm7j33l+6cWZyOfhZ/fjsvc1s++hLp/bcQ7kEtLyWHn170Nzix86/bXecKzldQub2b+nVP5T1K/9Cs2tq6qulxaWOPmf+cwYAy7UtAAgbdDvFhcWO4A5QdbaKuIhJjXZvV60GPDw1inoDfGVlJR999BHp6ekcP36ciooK/Pz8CA4OJiwsjEGDBuHtrf9PNFSnn3UA4MDBbKf2w4dz+H8d2uHp6Um1CTMOoyg5XcKKWUm12m8b2IeTx05wfevrAcg74vwmYn5OHmH31Ly2fvLYCXZ+uoPhsZHk7M2huPAMY2eNp6SohK8217zG3q5re3IPHuWOB/oy8qnRtA5pQ87eI7wxewWZ279t5Lu8ypjw71OdFfqcnBwefPBBnn/+eQ4ePIjFYqF169Y0b96c/fv3M336dIYMGcLRo/oxsKFaWC0AnDlT7NR+5kwxXl5e+PvX/QKDXH0G/noQt/brxft/SKN5Cz8qyyupOuu8sVVZcRnNW/zw3/6N2a/j18KPZX9/jTf/9Sduv/8O5k94kVN5NTsNBgQG0Lp9a6Kff5T3Xl3D3N+9wOl/n2bmqtnc0LbVFb2/Jq/a7vphEHWm3i+88AIhISG8//77WCyWWueLi4uJj48nISGBpKTa2YpcnIeHBwA/fsZ9rl3Zu7H84ld38/iLk9j61y18+OYHDIsZgZ3agcTDwwP7fwNMYFAgv38/kdOnTjN/wv9RVlzKPY/cyzOvT+eF0bPY96+9eDXz4tobrmP68Gf4bkfNxlNZO7/jtc9XMPTxYbw+47Urep9Nmgl/o1OdGXxGRgZTp069YHAHsFgsPPXUU+zcubNRJmdkRadraqktWjj/u7VY/LHZbJSUlF7oa3IVGjLuf5i86EnSN+1k4RMLACg9U0Izn2Z4eXs59fX196X0TAkAAyIH4h9gYc5vnmfbxq188/nXvDxpPt/vy+G3z/0OgPKScspLyx3BHaCirIK9X+2hXZd2V+YGrxYmzODrDPBWq5X8/Py6upCbm1vvfghS2/4DhwHoEHKzU3tIyM3s3XfQHVOSRjBq2m+Ifn48/0j7Oy89/n+Oksyxw8fx9PQk6KYgp/5BNweTezAXgJZtbuDU8ZP8O//fTn2y0rO46Wc1f26OZ9eM4+nl/FfZu5l3rZ8Ozc5eXe3yYRR1Bvjhw4fzzDPPkJqayqFDhygtLaWqqorS0lKys7N59913mT59Og8//PCVmq9h7N9/iJycXB566D5Hm7e3Nw/cH8HmzVvcODO5XAZHD2F4bCQb3vgLS55cRLXth8CxNyOLivIKwu693dHmH+BP9/Cfs+uLb4CaJZEt29zA9cHXO43bqVcn8r+vSby+/ue/8PH1oc/AMMd5P6s/nUO7sDdjT2Pe3tXHZnP9MIg6a/BxcXF4eHjw0ksvOba1PJ+/vz+jRo1i8uTJjTZBI3spcTlLFs+lsPA0W7fuZNLE39GyZSCLl6xw99TkEl3X6jp+88zvyM46zJb1n9OpV2en8wd27efDNz9g5NTR2KvtHDucy/DYSEqLS/k0tWbp7KZ3/8aQcQ8xc9Vs3l2cSumZUvoPG0CX27ry+0fnAfDN5//imy1fE5P4BC2us/Lv/FMMi6l5xf2D5PVX9qabOgOVXlzl0puslZWV7Nmzh/z8fMrKyvD19SU4OJguXbrUuxdCXcz+JitA/JQJxMWOo2XLQL75JpOnp83RVgUG8MvhETzxypSLnv9tz1GUFBUz6unf8MvhEfj6+7I3Yw8rn3fequCGG2/gdzOi6XnXreDhQXbWYd5esJrMbT8sgfT182X0//6WOwffha9/c/Z+tYfk2SvI2ZfTqPd4pV3qm6wlsx9xua//7Lcv6VpNhbYqkCbFKAFeLr9LDvCzfu1yX/85qZd0raZCbyiJiDmYcJmkAryImIMJa/AK8CJiCvYq46yOcZUCvIiYgzJ4ERGDUg1eRMSglMGLiBiTXQFeRMSg9JBVRMSglMGLiBiUAryIiDGZcftkBXgRMQdl8CIiBqUALyJiTPYqvegkImJM5ovvCvAiYg560UlExKgU4EVEDEolGhERY1KJRkTEoOxVCvAiIsZkwhKNp7snICJyJdirXT8aat26dTzwwAPccsstPPjgg2zcuNFx7ujRo0yYMIHevXvTt29fEhMTqaqqcvr+6tWriYiIoEePHkRFRbFr1y6n866McSEK8CJiDtUNOBrgL3/5C8899xxRUVF88MEHDB48mCeffJKMjAwqKysZN24cHh4epKamkpCQwNq1a1m6dKnj+2lpaSQmJjJlyhTS0tIICQlh/PjxnDp1CsClMS5GAV5ETKExMni73c7ixYsZPXo0Y8aMoV27dkycOJG+ffuybds2Pv74Y3Jzc5k/fz6dOnUiIiKCqVOnsmrVKsrLywFISkpi5MiRDBkyhI4dOzJv3jwsFgupqakALo1xMQrwImIK9irXD1cdOnSI3NxcBg8e7NT+xhtvEBMTQ3p6Ol27diUgIMBxLjw8nNLSUjIzMzl58iTZ2dmEh4c7znt5eREaGkp6ejpAvWPURQ9ZRcQUGpKZFxUVUVRUVKvdarVitVodn7Ozs4GaMspjjz3G7t27adu2LRMnTmTAgAHk5+cTHBzsNEarVq0AyMvLw9fXF+CCfXbv3g1Q7xh1UYAXEVNoSIBPSUlh2bJltdpjY2OJi4tzfC4uLgZg2rRpxMTEEB8fzyeffMKkSZN44403KC8vx9/f32kMHx8fACoqKigrK3NqO79PZWUlQL1j1EUBXkTMwe7hctcxY8YwdOjQWu3nZ+8AzZo1A2Ds2LEMGzYMgK5du/Ltt9+SnJyMr6+vI1Cfc+6zn5+fI4O/UB8/Pz+AeseoiwK8iJhCQzL4H5diLuZc6aRTp05O7T/72c/YtGkTd9xxB1lZWU7nCgoKHN9t06aNo61z585OfYKCghz96hqjLnrIKiKmYK/2cPlwVbdu3fD393fUy8/Zt28fN998M3369CErK8upnr99+3b8/f3p1q0bgYGBhISEsGPHDsd5m81GRkYGYWFhAPWOURcFeBExhWqbh8uHq3x9fRk/fjyvvvoq69evJycnh9dee40tW7YQHR3NwIEDCQoKIj4+nj179rB582YWLFjA2LFjHXX06OhoUlJSSEtL48CBA8yYMYOSkhJGjBgB4NIYF6MSjYiYwk95Q9UVkyZNws/PjyVLlpCXl0eHDh1YunQpd9xxBwArV65kzpw5REZGYrVaiYqKIiYmxvH9yMhIiouLWbx4MYWFhXTv3p3k5GQCAwMBuOaaa+od42I87G78VePePje669LSRA0J7u3uKUgT9X7Ohkv6/vd9Ilzue9POTZd0raZCGbyImIL7Uln3UYAXEVNoyMNTo1CAFxFTaMjDU6NQgBcRU1AGLyJiUPYGvMlqFArwImIKjbVMsilTgBcRU6hWBi8iYkwq0YiIGJRW0YiIGJRW0YiIGJRq8CIiBqUavIiIQWkvGhERg1KJRkTEoKr1kFXEvd79arG7pyAGpQxeRMSg9JBVRMSglMGLiBiUCRfRKMCLiDnYqj3dPYUrTgFeREzBhLsFK8CLiDnYUQ1eRMSQqk1YhFeAFxFTqFYGLyJiTCrRiIgYlE0BXkTEmLSKRkTEoBTgRUQMSjV4ERGDMuFuwQrwImIOZlwmab7NGUTElGwNOH6qw4cP06tXL9asWeNoO3r0KBMmTKB379707duXxMREqqqqnL63evVqIiIi6NGjB1FRUezatcvpvCtjXIgCvIiYQrWHh8vHT3H27FmmTp1KaWmpo62yspJx48bh4eFBamoqCQkJrF27lqVLlzr6pKWlkZiYyJQpU0hLSyMkJITx48dz6tQpl8e4GAV4ETEFewOOn2Lp0qX4+/s7tX388cfk5uYyf/58OnXqREREBFOnTmXVqlWUl5cDkJSUxMiRIxkyZAgdO3Zk3rx5WCwWUlNTXR7jYhTgRcQUqhtwNNTOnTt55513mD9/vlN7eno6Xbt2JSAgwNEWHh5OaWkpmZmZnDx5kuzsbMLDwx3nvby8CA0NJT093aUx6qKHrCJiCo21iqaoqIhp06YxY8YMWrdu7XQuPz+f4OBgp7ZWrVoBkJeXh6+vL8AF++zevdulMeqiAC8iptCQrQqKioooKiqq1W61WrFarU5ts2fP5tZbb2XIkCG1+peXl9cq2/j4+ABQUVFBWVmZU9v5fSorK10aoy4K8CJiCg3J4FNSUli2bFmt9tjYWOLi4hyf161bR3p6Ohs2bLjgOL6+vo5Afc65z35+fo4M/kJ9/Pz8XBqjLgrwImIKDamtjxkzhqFDh9Zq/3H2/t5773Hq1Cn69+/v1D5nzhzefPNN+vTpQ1ZWltO5goICoKYs06ZNG0db586dnfoEBQU5+tU1Rl0U4EXEFBqyOuZCpZgLefnll2utZBk0aBCxsbEMHjyYr7/+mrS0NIqKihzjbd++HX9/f7p164aPjw8hISHs2LGDfv36AWCz2cjIyCAqKgqAPn361DlGXbSKRkRModrD9cNVQUFBtGvXzukACAwM5MYbb2TgwIEEBQURHx/Pnj172Lx5MwsWLGDs2LGOOnp0dDQpKSmkpaVx4MABZsyYQUlJCSNGjABwaYyLUQYvIqbgjt0kr7nmGlauXMmcOXOIjIzEarUSFRVFTEyMo09kZCTFxcUsXryYwsJCunfvTnJyMoGBgS6PcTEedrvdbb+p0NvnRnddWpqosmOfu3sK0kQ1a9nhkr7/h5tGu9z38e//dEnXaiqUwYuIKWg/eBERg1KAFxExKLfVot1IAV5ETEG/8ENExKBUohERMahL+UUeVysFeBExBZVoREQMSiUaERGD0ioaERGDqjZhiFeAFxFT0ENWERGDUg1eRMSgtIpGRMSgVIMXETEo84V3BXgRMQnV4EVEDMpmwhxeAV5ETEEZvIiIQekhq4iIQZkvvIOnuydgduOiR5KVuYUzpw+w5Z/ruT081N1TksuktLSMeQte5ReDHyFs4MM8Fj+dPfsPOc6f+k8h//vCS/S9bwR97xtB/PS55B7Pv+h48xcnMenp5+u85o6vdnHLXQ+w46tdl+0+jKK6AYdRKMC70ejRw3l1+e/589vvERn1GIWFp/nwr6tp3/4md09NLoMp0+eybuPfGDtyGAvnTadl4HWMmTSVw0eOcvbsWcY/8Sxbd2QQP3EsL83+X8rLK/jN409ReLqo1lh/Xruet95dV+f1yisqeP73i7DbzZir1s+G3eXDKFSicaPZs6ayYuVqEuYuBOBvn/6T7779J5OfeJT4J2e5eXZyKTL37Gfrjq+Y9XQckb96AIA7w0MZNeFJlq1cxX0DfsH+Q9kkvTKXO//7U1tY7x4MfuRR3vjTGp6KGQfUZPkLX01m/UebaGHxr/OaS19fRWVlZePe2FXMjDV4ZfBu0rFjCO3b38QHH3ziaKuqquLDjZu4995funFmcjkc+T4XwBG8z+l1Sze+2J5B9ve5eHl5Eh56q+Ocj48PP+/aiS3b0x1tK1JS+WpXJkmvzKXzzzpc9Hq7v9vLu+v+ytTYRy/znRiHvQGHUSjAu0mn//5lPXAw26n98OEc/l+Hdnh66j/N1Sy4VUsAjucXOLXnHs+juKSUVjdcj81WzYmTp5zOHz2Wz7Hz6vBRQwezfvXr3NGn10WvdfbsWWa+uJDxv40i5Oa2l/EujKUau8uHUSiKuEkLqwWAM2eKndrPnCnGy8sLf38/d0xLLpOfd+1E+5tuZO7Ly/k2ax9FZ4p5d92HfP5lTXYe1rsn111r5dmERA5m51B4uojlb/yJA4ezKSuvcIwT0q4t3t5edV4r6c238fD0IHrUiEa9p6udGR+y1luDHzlyJB4erm3Dtnr16kuekFmc+3f64wdi59qrq430x8x8fHx8WPTiTKa9MJ9fj58MQM+fd2XsqOG8lrya5r7XsPjFmTyb8DL/M2oCAHffGcbwIfex7sNPXb7OvoOH+eOf3+PN5S/RzFuP1OpiN1Bm7qp6/0TcfffdLFq0iA4dOtCjR48rMSdTKDp9BoAWLSwUFJx0tFss/thsNkpKSt01NblMOnZoR1rKqxzPP4HNZqNtm2BeTV6Np6cnFn9/evf8OR+t+SNHj+Xh49OMoBtaMmPeKwRYW7g0vs1mY+aLCxn20H107dSRqiobtv8mBtXVNmw2G15edWf/ZmKk1TGuqjfAT5gwAYvFwoIFC0hKSqJtW9X4Lof9Bw4D0CHkZg6eV4cPCbmZvfsOumlWcrmUlZfzt8++4PbQW2kddIOjfd+Bw3Ts0I4zxcX844sdDPrlXdx0Y+sfzh88TJc6HqaeL6/gJJl79pO5Zz9/Xrve6dz4yc9xW69beHPZS5fnhgzAjD8Tu1SDHzVqFGFhYSxatKix52Ma+/cfIicnl4ceus/R5u3tzQP3R7B58xY3zkwuB29vbxISl7Jx0z8cbUeP5fH5lzvpf2c4Z89WMePFV/hix1eO819/m8V3ew9w953hLl2jVctAUlcudjrmPz8NgFlPx/H8009c3pu6ylXb7S4fRuFy0W7OnDlkZmY25lxM56XE5SxZPJfCwtNs3bqTSRN/R8uWgSxessLdU5NL1Mzbm4eH3MfrKakEXnctFj8/XnktmeuuC+C3UUO5NsDKL++6nZeXrcDDA86erWL+kiQ6d+zA/9wf4do1mjXj5107ObV5/7ck0/7mtoS000/b5zNO2HadywG+VatWtGrVqjHnYjp/SEqheXNf4mLHMfmJR/nmm0weeHAUhw/nuHtqchnETxyLB7Bg+UoqK88S1rsnT8WM49oAKwBzpz/J7xcn8fzvF+Pp6cHdfcN5clI0zZo1c+/EDcpIyx9d5WF343vN3j43uuvS0kSVHfvc3VOQJqpZS9eeTVzMI+1+5XLft4/UvS3E+YqLi1myZAmffvop//nPfwgJCSEmJoaIiJqfxI4ePUpCQgI7d+7E19eXoUOHEh8fj/d5q55Wr15NcnIyJ06coGvXrkyfPt1pUYsrY1yI1sGLiClUYXf5aIhnn32Wzz77jLlz57Ju3ToGDRpEbGwsX375JZWVlYwbNw4PDw9SU1NJSEhg7dq1LF261PH9tLQ0EhMTmTJlCmlpaYSEhDB+/HhOnap5Cc6VMS5GGbw0Kcrg5WIuNYMf3u4hl/uuPbK+/k7AiRMnuOuuu0hKSqJ///6O9jFjxtCyZUv69+/Ps88+yxdffEFAQAAAa9as4cUXX+TLL7/E19eXe++9l4iICKZNq3lAbrPZuOeeexg2bBgxMTFs2LCh3jEuRhm8iJhCY7zJ2rx5c1asWMFtt93m1O7h4cHp06dJT0+na9eujsAMEB4eTmlpKZmZmZw8eZLs7GzCw39YOeXl5UVoaCjp6TVvPdc3Rl306puImEJDihVFRUUUFdXettlqtWK1Wh2fLRYLv/jFL5z6fP3112zbto0ZM2awZcsWgoODnc6fW6ySl5fnyL4v1Gf37t0A5Ofn1zlGXRTgRcQUGrKKJiUlhWXLltVqj42NJS4u7qLfO3jwILGxsfTs2ZOoqCg+/fRT/P2dt3n28fEBoKKigrKyMqe28/uc2/q5vLy8zjHqogAvIqbQkK0KxowZw9ChQ2u1n5+9/9jOnTuJjY2lTZs2JCUl0axZM3x9fWvt0X/us5+fnyODv1AfP7+aDQfrG6MuCvAiYgoNyeB/XIqpz/r163nuuecICwtjyZIlWCw1u8UGBweTlZXl1LegoMBxrk2bNo62zp07O/UJCgpyaYy66CGriJiC3W53+WiIDRs2MG3aNO6//36SkpIcwR2gT58+ZGVlOdXzt2/fjr+/P926dSMwMJCQkBB27NjhOG+z2cjIyCAsLMylMeqiAC8iptAYq2jy8vKYOXMm4eHhPP300xQWFnLixAlOnDhBYWEhAwcOJCgoiPj4ePbs2cPmzZtZsGABY8eOddTRo6OjSUlJIS0tjQMHDjBjxgxKSkoYMaJmf39XxrgYlWhExBQaYz/4Tz75hLKyMrZt20a/fv2czvXu3Zu3336blStXMmfOHCIjI7FarURFRRETE+PoFxkZSXFxMYsXL6awsJDu3buTnJxMYGAgANdcc029Y1yMXnSSJkUvOsnFXOqLTgNvutflvp9+//ElXaupUAYvIqZgs5tvR3gFeBExBf3KPhERgzLSL/JwlQK8iJiC+cK7AryImIQZf+GHAryImIICvIiIQWkVjYiIQWkVjYiIQbnxnU63UYAXEVNQDV5ExKCUwYuIGJStQftEGoMCvIiYgt5kFRExKK2iERExKGXwIiIGpQxeRMSglMGLiBiUtioQETEolWhERAzKrgxeRMSYtFWBiIhBaasCERGDUgYvImJQtmrV4EVEDEmraEREDEo1eBERg1INXkTEoJTBi4gYlB6yiogYlEo0IiL+JYIEAAAC2ElEQVQGpRKNiIhBmXG7YE93T0BE5EqwN+CfhqiurmbJkiX069ePnj17Eh0dzZEjRxrpLhpGAV5ETKHabnf5aIjly5fz9ttvM3fuXN555x28vLwYN24cFRUVjXQnrlOAFxFTqLZXu3y4qrKykuTkZGJjY7n77rvp0qULCxcu5OTJk2zcuLER78Y1CvAiYgp2u93lw1VZWVmUlpZy++23O9osFgvdunUjPT29MW6jQfSQVURMoSGBu6ioiKKiolrtVqsVq9Xq+Jyfnw9AUFCQU79WrVpx/PjxnzjTy8etAb6qMtedlxcREznbgHizdOlSli1bVqs9NjaWuLg4x+eysjIAfHx8nPr5+PhQWVn5E2d6+SiDFxH5kTFjxjB06NBa7edn7wC+vr5ATS3+/CBfWVmJn59f407SBQrwIiI/8uNSzMW0bt0agIKCAiwWi6O9oKCAjh07Ntr8XKWHrCIiP1GXLl2wWCzs2LHD0VZcXMx3331HWFiYG2dWQxm8iMhP5OPjw+jRo1m4cCEtW7akbdu2LFiwgKCgIAYNGuTu6SnAi4hciieeeAKbzcasWbMoKysjNDSUlStX1nrw6g4edjPuwCMiYgKqwYuIGJQCvIiIQSnAi4gYlAK8GzXlbUalaUhKSuKRRx5x9zTkKqUA70ZNeZtRcb/Vq1ezcOFCd09DrmIK8G7S1LcZFffJz8/n8ccf5+WXXyYkJMTd05GrmAK8mzT1bUbFfTIzM/H392f9+vX07NnT3dORq5hedHKTpr7NqLjPgAEDGDBggLunIQagDN5Nmvo2oyJy9VOAd5Pztxk9X1PZZlRErn4K8G5y/jaj5ysoKKhVthER+SkU4N2kqW8zKiJXPz1kdZOmvs2oiFz9FODdqClvMyoiVz9tFywiYlCqwYuIGJQCvIiIQSnAi4gYlAK8iIhBKcCLiBiUAryIiEEpwIuIGJQCvIiIQSnAi4gY1P8HYCzbuVlNXP0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "best_alpha=5\n",
    "testmulnb(train_bow,train_y,test_bow,test_y,best_alpha) #BoW testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tPositive\t\t\t\t\tNegative\n",
      "________________________________________________________________________________________________\n",
      "\t-21.2789\tabhor          \t\t\t\t-4.3587\tteeco          \n",
      "\t-21.2789\tabod           \t\t\t\t-4.3872\tlitmus         \n",
      "\t-21.2789\taborb          \t\t\t\t-4.5554\tgrade          \n",
      "\t-21.2789\taboveour       \t\t\t\t-4.5573\tlustrous       \n",
      "\t-21.2789\tabras          \t\t\t\t-4.5635\tgrown          \n",
      "\t-21.2789\tabsenc         \t\t\t\t-4.6280\tflower         \n",
      "\t-21.2789\tabsoluet       \t\t\t\t-4.6665\tprotool        \n",
      "\t-21.2789\tabysm          \t\t\t\t-4.7603\tvegetariani    \n",
      "\t-21.2789\tacademia       \t\t\t\t-4.7763\tordor          \n",
      "\t-21.2789\tacana          \t\t\t\t-4.8800\ttryptophan     \n",
      "\t-21.2789\taccept         \t\t\t\t-4.9699\tcollar         \n",
      "\t-21.2789\taccostom       \t\t\t\t-4.9846\ttenderloin     \n",
      "\t-21.2789\tachiev         \t\t\t\t-5.0820\tgiraff         \n",
      "\t-21.2789\tacia           \t\t\t\t-5.0837\tmanipularlo    \n",
      "\t-21.2789\tactor          \t\t\t\t-5.1821\tbyt            \n",
      "\t-21.2789\tacupunctur     \t\t\t\t-5.2621\tyen            \n",
      "\t-21.2789\tadd            \t\t\t\t-5.2808\tamac           \n",
      "\t-21.2789\taddison        \t\t\t\t-5.2880\toutburst       \n",
      "\t-21.2789\taddit          \t\t\t\t-5.3209\tprodduct       \n",
      "\t-21.2789\tadio           \t\t\t\t-5.3328\tforta          \n"
     ]
    }
   ],
   "source": [
    "def show_most_informative_features(vectorizer, clf, n=20):\n",
    "    feature_names = uni_gram.get_feature_names()\n",
    "    \n",
    "    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))\n",
    "    top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])\n",
    "    print(\"\\t\\t\\tPositive\\t\\t\\t\\t\\tNegative\")\n",
    "    print(\"________________________________________________________________________________________________\")\n",
    "    for (coef_1, fn_1), (coef_2, fn_2) in top:\n",
    "        print(\"\\t%.4f\\t%-15s\\t\\t\\t\\t%.4f\\t%-15s\" % (coef_1, fn_1, coef_2, fn_2))\n",
    "        \n",
    "show_most_informative_features(train_bow,mulbnb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Best alpha Value 0.01 with highest roc_auc Score is 61.25504009248076\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEgCAYAAABFO1+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XlclOXeBvBrhtkYZoTBWE5hLiQKCIgmiqmhBKXI6Zjl0qsetyOh5jlqr6ZtdkLL40sWlqYpttgprTyox6VyK3fFJdFQTEXFEAhRQJaBmef9A5kYAWcGGAZmru/nwyfnWWZ+N+lcc9/3M88tEgRBABER0X2IbV0AERG1fAwLIiIyiWFBREQmMSyIiMgkhgUREZnEsCAiIpMYFkREZBLDgoiITGJYELVif//73/Huu+/augxyAAwLolaqsrISBw8exIABA2xdCjkAhgVRK3Xy5EmIRCKEhoY2+rm+++47DB8+HMHBwYiKisL3338PABg7diwWLFjQ6Oen1o9hQS3SoEGD8M4772DSpEkICQnBP/7xD+h0Ovz73/9GbGwsQkJC8Pjjj2Px4sUoKysznCcIAj7//HPExMQgODgYkZGR+PDDD6HT6Sx6/bKyMiQmJiI6OhrdunVDjx49MGHCBPzyyy+16nzppZeMth08eBBdunTBkSNHGlTT66+/jj59+qCystJo+wcffICQkBAUFxcDAPbu3Yt+/frByckJAJCeno6JEyeiV69e6N69O55//nkcOHDAZFuXLVuGGTNmoEOHDpg7dy6USiXmzZuHo0eP4sSJE5g8ebJ5vzSyawwLarHWrVsHHx8fLFu2DKNGjcLrr7+OhIQEDBgwAElJSRg9ejS++OILvPDCC6i+H+bSpUuxcOFChIeHY9myZRg5ciRWrFiBpUuXWvTac+fOxYYNGzBx4kR8/PHHmDNnDjIyMjBz5kxYeu9NS2t6+umnUVBQUOuNfuvWrYiMjIRKpQIA/PTTT4YhqOLiYkyYMAFyuRxLlizBe++9B6lUiri4OFy7dq3e2k6cOIEPPvgAY8eORWJiIv7nf/4H8+bNQ3FxMd566y0MHToUPj4+FrWX7JPE1gUQ1Uej0eC1116DRCLBr7/+im+++QYvvvgipk+fDgB4/PHH4eXlhZdffhm7d+9GWFgYkpOTMWrUKLz66quGY+7cuYPDhw9Dr9dDLDb9+Uir1aKwsBDz58/H008/DQAIDw9HSUkJFi9ejJycHHh7e5vVhqKiIotr6tmzJx5++GFs3boVjz/+OADg7NmzuHTpEl5++WUAQHZ2Nn799VdDWFy8eBEFBQX429/+hh49egAAgoODsXLlSqOe170+/fRTuLi44O9//7th20MPPQQA+PXXX/Hee++Z1U6yfwwLarF8fX0hkVT9FT169CgAIDY21uiY2NhYvPLKKzh69ChkMhkqKirw5JNPGh0zc+ZMi15XJpNh7dq1AIDff/8dmZmZuHz5Mvbu3QugKkzMderUqQbV9PTTTyM5ORllZWVQKBTYsmULPDw80K9fPwBVQ1BBQUFwd3cHAHTu3BkeHh6Ij4/HkCFD0K9fP/Tu3Rvz5s2r9zV0Oh3279+P/v37Q61WG7aLRCIAQFRUFHx9fc1uK9k3DkNRi/XAAw8Y/nz79u1a2wBAIpFAo9GgsLAQBQUFdR7TEAcPHkRsbCwee+wxxMXFISUlBTKZDAAsGoZqaE1PP/00SkpKsGfPHuj1emzbtg1Dhw41zE/8+OOPRldBKZVKfPnll4iKisJ3332HqVOnIjw8HLNnz0ZRUVGdr3H16lUUFxcjMDCwzprHjBljUc1k3xgW1Cq4uroCqPqkX1NFRQUKCgqg0WjQpk0bAMDNmzeNjsnNzcWhQ4fuOxxT09WrV/HCCy+gc+fO2LlzJ1JTU/HFF19g4MCBdR6v1+uNHldPQANocE3t2rVDz549sX37dhw9ehQ5OTmGITGtVosjR44gIiKi1jkJCQk4cOAAUlJSMH78eGzfvr3euZFbt24B+ON3W23lypUAqoYBiaoxLKhVCAsLAwBs2bLFaPvWrVuh0+nQs2dPBAcHQyqV4ocffjA65osvvkB8fLzZPYIzZ86gvLwcEydORLt27QzDMj/99BMA43BQqVT47bffjM6vHjID0Kia/vKXv2Dfvn3YtGkT/Pz84O/vDwA4cuQIlEqlUY9gx44d6NOnD3JzcyESieDv74/Zs2ejU6dOuH79ep3PXx1kGRkZhm0//vijoVZzw5UcA+csqFV45JFHMGzYMCxfvhxlZWUICwvDuXPnsHz5cvTq1QsRERFwcnLCuHHjsHbtWkilUvTt2xfp6elYs2YN4uLi4OzsbNZrBQYGQiKRIDExEePHj0dFRQU2btyIH3/8EQBQWlpqOHbQoEFYsWIFkpKS8Oijj+Lw4cPYsWOHYb+7u3uDaxo8eDASEhKwadMmzJ4927C9egiqOsQAoEePHhAEAS+88AImT54MV1dX7Nu3DxcuXKj30tdOnTqhQ4cO+Oqrr6BSqaBQKLBq1So89dRT2LFjB1auXIkpU6YgODjYrN8b2TmBqAUaOHCgMHv2bKNtlZWVwvLly4XIyEghMDBQGDhwoLBkyRKhtLTUcIxerxeSk5OFqKgoITAwUIiOjhbWrl0r6PV6i15/+/btQkxMjBAUFCT069dPmDZtmnD06FGhS5cuwqeffmo4rqSkRHjjjTeEsLAwoXv37sILL7wgHD9+XPDz8xMOHz7c6Jpmzpwp+Pv7Czk5OYZtUVFRwrZt22ode+bMGWHy5MlC7969hW7dugmxsbHCN998c9/nT09PF0aOHCl069ZN6Nmzp7Bw4UKhsrJSWLRokdCtWzdhxYoV5v7KyM6JBMHCi8aJiMjhcBiKHIZer0dFRYXJ45ycnAyX7BJRFf6LIIdx7NgxjBs3zuRx06dPx4svvtgMFRG1HhyGIodRXFyMy5cvmzzO09MTXl5ezVARUevBsCAiIpP4PQsiIjLJLuYsCgruQK+3vIPUtq0K+fnFpg+0I2yzY2Cb7V9j2isWi6DRuFh0jl2EhV4vNCgsqs91NGyzY2Cb7V9ztpfDUEREZBLDgoiITGJYEBGRSQwLIiIyiWFBREQmMSyIiMgkhw2LrYcy8ey8/9q6DCKiVsFhw+LbHy+hXKtDUYnW1qUQEbV4DhsWSnnV9xGLS03fspqIyNE5bFgo5E4AgDullTauhIio5XPYsJBLq8KiuIw9CyIiUxw2LJzvDkPd4TAUEZFJDhsWCtndYagyDkMREZnisGFRPQxVwmEoIiKTHDYsqnGCm4jINKutZ3HkyBGMGzeuzn0+Pj7YtWsXEhMTsWrVqlr7z549C4mkeZbauMOeBRGRSVZ7Rw4NDcX+/fuNtmVkZGDKlCmIi4sDAJw/fx4jRozAjBkzjItqhqCoXnmcV0MREZlmtXdlmUwGDw8Pw+OKigosWrQIUVFRGDFiBICq8Bg4cKDRcc2Nw1BERKY127Kqn3/+ObKzs5GcnAwAKCwsRHZ2Nh555JHmKqFOHIYiIjKtWSa4S0tLsXLlSowbNw5eXl4AqnoVALBlyxZER0dj4MCBmDt3LnJzc5ujJIMSXjpLRGRSs/QsNm3ahPLycqMJ7+qwUKlUSEpKQl5eHpYuXYqxY8ciJSUFzs7OZj9/27Yqi2uSyv64dLZtWxXEYpHFz9FaeXiobV1Cs2ObHYOjtbk529tsYREVFQV3d3fDttGjRyMmJgaurq4AgK5du8LPzw+PP/44du7cidjYWLOfPz+/GHq9YFFN2vKqHoVeAK5dL4BSIbXo/NbKw0ONvLwiW5fRrNhmx+BobW5Me8VikcUfsq0+DHXz5k2cOnUKQ4cONdouEokMQVHNy8sLbm5uyM7OtnZZRoo5FEVEdF9WD4sTJ05AJBKhV69eRtsTEhLwl7/8xWjbtWvXUFBQ0CyT3jX7Ibw/FBHR/Vk9LH755Re0a9cOSqXSaPtTTz2FCxcuICEhAZmZmTh69CimT5+O4OBgREREWLssI7wiiojo/qweFnl5ebWGmwDg0UcfxUcffYS0tDQMGzYML774IgIDA7Fq1SqIxc1zFxKZpOp1eEUUEdH9WX2C+6233qp3X//+/dG/f39rl1AvpbMU2qJylGl1NquBiKg1cOgbCbooqrKytJw9CyKi+3HosFDIGRZEROZw6LAQiURQyJxQWs5hKCKi+3HosACqllct1bJnQUR0PwwLuQRlHIYiIrovhoXMiXMWREQmOHxYKOQSlPLSWSKi+3L4sGDPgojItGZb/KilUcicAJGoas6CPQsiovty2LB4/gk/uGmU+Pf2dJSwZ0FEdF8OOwzVxkWGtq7OUMicUK7VWbweBhGRI3HYsKjmfPdb3ByKIiKqH8PCEBYciiIiqg/D4m5YcN6CiKh+DAuZEwCgjPeHIiKql8OHheHOsxyGIiKql8OHhTNvU05EZBLD4u4wFMOCiKh+DAtDz4JzFkRE9XH4sJBXT3BzzoKIqF4OHxZirpZHRGSSw4cFcHe1PM5ZEBHVi2EBLq1KRGQKwwJVV0RxaVUiovpZ7RblR44cwbhx4+rc5+Pjg127diErKwtvvfUWjh07BoVCgWHDhmHmzJmQSJr3zukKDkMREd2X1d6VQ0NDsX//fqNtGRkZmDJlCuLi4qDVajFp0iR07NgRX331Fa5du4b58+dDIpFg5syZ1iqrTkq5BL/fLmvW1yQiak2sNgwlk8ng4eFh+HFzc8OiRYsQFRWFESNG4LvvvsP169exePFi+Pn5ITIyEi+99BI+++wzlJU17xu3q0qGW0XlEASuaUFEVJdmm7P4/PPPkZ2djXnz5gEAUlNT4e/vD1dXV8MxvXv3RklJCc6ePdtcZQEA3NUKlFfoePksEVE9miUsSktLsXLlSowbNw5eXl4AgJycHHh7exsd5+npCQC4ceNGc5Rl4KaWAQAKisub9XWJiFqLZplJ3rRpE8rLy40mvMvKyuDi4mJ0nExW9aZdXm7Zm3bbtqoG1+bhoUandloAgF4shoeHusHP1Vo4QhvvxTY7Bkdrc3O2t9nCIioqCu7u7oZtCoUCWq3W6Ljqx0ql0qLnz88vbtAa2h4eauTlFQG6quGnzKwCtHN3tvh5WhNDmx0I2+wYHK3NjWmvWCyy+EO21Yehbt68iVOnTmHo0KFG2729vZGbm2u0rfrxvcNT1qZRVfVobhVxGIqIqC5WD4sTJ05AJBKhV69eRtt79eqF9PR0FBYWGrYdOXIELi4uCAgIsHZZRqQSJ6icpShgWBAR1cnqYfHLL7+gXbt2tYaWnnjiCXh5eWHmzJk4d+4cdu/ejcTEREyYMMEwd9Gc3NVyhgURUT2sHhZ5eXlGl8dWk8vlWL16NQBgxIgReP311zFy5EhMmzbN2iXVyY1hQURUL6tPcL/11lv17mvfvj3WrFlj7RLMolHLcTm70PSBREQOiDcSvEujlqOopAIVlXpbl0JE1OIwLO7SqOQAgFv8Yh4RUS0Mi7s0barCgvMWRES1MSzuqu5ZMCyIiGpjWNylUSsAMCyIiOrCsLjLWe4EudSJYUFEVAeGxV0ikQgatZx3niUiqgPDogaNWo6CIq6YR0R0L4ZFDRq1nDcTJCKqA8OiBo1ajlvFWui5vCoRkRGGRQ0atRw6vYCiO1rTBxMRORCGRQ3V37W4yaEoIiIjDIsa3NR3b/nBsCAiMsKwqMFdzZ4FEVFdGBY1qF1kcBKLeDNBIqJ7MCxqEItEcFPJcLOQYUFEVBPD4h5uajl7FkRE92BY3EOjVnDOgojoHgyLe2hUVd/iFvjFPCIiA4bFPTRqOcordCgt19m6FCKiFoNhcQ+NunoRJN5QkIioGsPiHoaw4CQ3EZEBw+IehrDg5bNERAYWhcWxY8cAALdu3cLOnTvNOiclJQVDhgxBUFAQYmJisH37dsO+xMREdOnSpdZPZWWlJWU1KTcVexZERPeSmHvg0qVLceLECXz++ecoKyvDqlWrkJGRgalTp9Z7zqZNmzB//nzMnTsXERER2LZtG2bNmgVPT0/07NkT58+fx4gRIzBjxgzjoiRml9XkpBIx1Eopl1clIqrB7J7Frl27kJycDADw9vbGunXrsG3btnqPFwQB77//PsaMGYO//vWvaN++PeLj49G3b18cPnwYAJCRkYGAgAB4eHgY/diaRiVnWBAR1WD2R/iKigpIpVLDY6lUCpFIVO/xly5dwvXr1zF06FCj7WvWrAEAFBYWIjs7G4888oilNVudRi3nF/OIiGowu2fRo0cPzJ49G4cOHcLhw4cxb948hISE1Ht8ZmYmAECr1WLKlCkIDw/Hc889h927dwOo6lUAwJYtWxAdHY2BAwdi7ty5yM3NbURzmkbVWtwMCyKiaiLBzK8ql5SU4P3338ehQ4cgkUgQHh6O6dOnw9nZuc7jN23ahDlz5uChhx7CtGnTEBAQgO+//x4rVqzAmjVrcOXKFbz55puYNGkS/vznPyMvLw9Lly7FnTt3kJKSUu/zNof1P5zHuh3n8O07QyGTOtmsDiKilsLsYSilUol58+aZ/cTVQ1YTJkzA8OHDAQD+/v44c+YMkpOTsXr1asTExMDV1RUA0LVrV/j5+eHxxx/Hzp07ERsba/Zr5ecXQ6+3/PYcHh5q5OUV1a79bn/rQmY+PN1sF1rWUF+b7Rnb7Bgcrc2Naa9YLELbtiqLzjE7LF544YU6t3/00Ud1bvf29gYA+Pn5GW3v3Lkzdu3aBZFIZAiKal5eXnBzc0N2dra5ZVmFu1oBoGrFPHsLCyKihjA7LJ588knDnysqKvDdd9+hW7du9R4fEBAAFxcXpKWloXfv3obtGRkZePjhh5GQkIDU1FSkpKQY9l27dg0FBQU2n/R2M6yYx1t+EBEBFoTFsGHDaj0eO3ZsvccrFApMnjwZy5cvh6enJ7p3746tW7di//79WLt2LaRSKb788kskJCRgzJgxyM3NxcKFCxEcHIyIiIgGN6gpuBvW4tbatA4iopaiwd9+EwTB5JVLU6dOhVKpRFJSEm7cuIFOnTph2bJlCA8PB1A1hPXBBx9g2LBhkMlkiIyMxP/+7/9CLLbtXUic5RLIZU7sWRAR3dXgOYuMjAyEhYWZPG/8+PEYP358nfv69++P/v37m1tCs3JXV61rQUREDZyzEIlEGD16NPr162eVoloCN36Lm4jIoMFzFoIg4MqVK+jQoUNT19QiaNRynLtaYOsyiIhaBLPD4quvvsK//vUvlJaWGra5u7vjwIEDVinM1jRqOW4VaaHXCxCL67+tCRGRIzA7LFatWoW1a9dixYoV+Mc//oE9e/bgxo0b1qzNpjRqOfSCgMISreG25UREjsrsy47c3NwQEhICf39/5OfnIz4+3rC+hT36Y3lVzlsQEZkdFhKJBLdv30b79u1x+vRpAIBOp7NaYbbGsCAi+oPZYTFixAjExcUhIiIC69evxzPPPANfX19r1mZTmru3/GBYEBFZMGfx7LPPYsiQIVAqlVi/fj3S0tIMl87+97//rbVuRWunVkrhJBYxLIiIYOEa3EqlEkDVDf+eeOIJKBRVn76rFzSyJ2KRiN+1ICK6q0nuq2HmkhitTtUiSLzlBxFRk4TF/ZZXbc00ajkKinkzQSIi296xr4Wr7lnYa8+JiMhcDIv70Kjl0FboUVpeaetSiIhsinMW96ExLILESW4icmwWhUX1N7Zv3bqFnTt3GrZbsl52a1J9mw/eqpyIHJ3ZYbF06VIkJSUBAMrKyrBq1SosX74cADBp0iTrVGdj7uxZEBEBsCAsdu3aheTkZACAt7c31q1bh23btlmtsJbATc2eBRERYEFYVFRUQCqVGh5LpVK7vWS2msRJjDZKKXsWROTwzL7dR48ePTB79mw8++yzEIlESElJQUhIiDVraxHc1HLcKmZYEJFjM7tn8dprr+GBBx7A22+/jX/9619o27YtXnnlFWvW1iK4qxW85QcROTyzexZKpRLz5s1DVlYWdDod2rdvb826Wgw3tRy/Xr9t6zKIiGzK7LDIzMzEtGnTkJubC0EQ4ObmhpUrV9r1bcqBqu9aFJdWoKJSB6nEydblEBHZhNnDUG+99RYmT56MY8eOITU1FfHx8XjzzTetWVuLoFFxESQiIrPDIj8/H8OGDTM8Hj58OAoKCqxSVEuiacOwICIyOyx0Oh1u3bpleHzz5k2zzktJScGQIUMQFBSEmJgYbN++3bAvKysLcXFx6NGjB/r27YslS5agsrJl3YeJPQsiIgvmLMaMGYORI0di8ODBEIlE2LZtG/7617/e95xNmzZh/vz5mDt3LiIiIrBt2zbMmjULnp6eCAoKwqRJk9CxY0d89dVXuHbtGubPnw+JRIKZM2c2umFNxbAWNy+fJSIHZnZY/Pvf/8abb76Jffv2Qa/X44033kDfvn3rPV4QBLz//vsYM2aMIVTi4+ORmpqKw4cP47fffsP169exYcMGuLq6ws/PDy+99BIWLVqE+Ph4wyp8tuYsl0Ahc0JBIcOCiByX2WHh7OyMDh06oE+fPmYdf+nSJVy/fr3W2tzVS7C+8cYb8Pf3h6urq2Ff7969UVJSgrNnz6Jnz57mlmZ1VYsgMSyIyHGZHRalpaWIjIyEt7e3YS1uANiyZUudx2dmZgIAtFotpkyZgrS0NPj4+CA+Ph6DBg1CTk4OvL29jc7x9PQEANy4ccOiRrRtq7Lo+Jo8PNQmj/Fyd0FxaaVZx7YG9tIOS7DNjsHR2tyc7TU7LCz9tnZxcTEAYM6cOZg2bRpmzpyJ77//HlOnTsWaNWtQVlYGFxcXo3NkMhkAoLzcsk/x+fnF0OstX1PDw0ONvLwik8e5yJ1w5UahWce2dOa22Z6wzY7B0drcmPaKxSKLP2SbHRZhYWEWPXH1TQcnTJiA4cOHAwD8/f1x5swZJCcnQ6FQQKs1Xt+6+nHNnktLoGkjx+1iLfR6AWKxfd88kYioLlZbVrV6iMnPz89oe+fOnZGVlQVvb2/k5uYa7at+fO/wlK1pVHLoBQG372hNH0xEZIesFhYBAQFwcXFBWlqa0faMjAw8/PDD6NWrF9LT01FYWGjYd+TIEbi4uCAgIMBaZTWIRl11ZRbvPktEjspqYaFQKDB58mQsX74cmzdvxtWrV7FixQrs378fEydOxBNPPAEvLy/MnDkT586dw+7du5GYmIgJEyYY5i5aCsNa3Lx8logclNlzFg0xdepUKJVKJCUl4caNG+jUqROWLVuG8PBwAMDq1avxz3/+EyNGjECbNm0wcuRITJs2zZolNUh1WLBnQUSOyqphAQDjx4/H+PHj69zXvn17w/cuWjKVUgonsQg3i8psXQoRkU1YbRjKnohFImjUcq7FTUQOi2FhJje1nDcTJCKHxbAwkzvDgogcGMPCTG6qqrAQBMu/KU5E1NoxLMykUcuhrdSjpLxlrbdBRNQcGBZmMqxrwe9aEJEDYliYiYsgEZEjY1iYyRAWnOQmIgfEsDCTG9fiJiIHxrAwk8RJjDYuMoYFETkkhoUFNCp+14KIHBPDwgIafjGPiBwUw8ICVWHBmwkSkeNhWFhAo5bjTlkltBU6W5dCRNSsGBYW4HctiMhRMSwsYFgEifMWRORgGBYWMCyvyrAgIgfDsLBA9Rfz2LMgIkfDsLCAs1wCZ7kTexZE5HAYFhZyU3F5VSJyPAwLC7mr5bwaiogcDsPCQlyLm4gcEcPCQhq1AreLtdDp9bYuhYio2Uis+eSXLl3C4MGDa21PSEjAc889h1mzZmHr1q1G+7y8vPDTTz9Zs6xG0ajl0AsCCu9UGC6lJSKyd1YNi/Pnz0OlUmHHjh1G29VqtWH/jBkzMGLECMM+Jycna5bUaDUXQWJYEJGjsGpYZGRkwNfXFx4eHrX2abVaZGZmIigoqM79LZXGsAhSGYA2ti2GiKiZWHXO4vz58/D19a1z38WLF1FZWYlHHnnEmiU0OU0brphHRI7HqmGRkZGB3NxcjBo1Cn379sXzzz+P/fv3A6gKEolEgpUrV2LQoEGIiopCQkICioqKrFlSo6mdpZA4iRgWRORQrDYMVVJSgqysLLi7u2P27NlwcXHB5s2bMXnyZCQnJ+PChQsAAB8fH3z00Ue4cuUKFi9ejHPnzuGzzz6DWGx+jrVtq2pwnR4eaovPaevqjNIKfYPObQlaa92NwTY7Bkdrc3O2VyQIgmCtJ79z5w6kUilkMplh26RJkyAIAlavXo3i4mK0afPHuP/JkycxatQofPnll+jRo4fZr5OfXwy93vJmeHiokZdneU/m7XXHIRaJMPd/zK+xpWhom1szttkxOFqbG9NesVhk8Ydsqw5Dubi4GAUFAPj5+eG3336DWCw2CgoA6NKlCwAgOzvbmmU1mobf4iYiB2O1sDh58iRCQ0Nx+vRpo+1nzpxB586dMW3aNMTHxxvtqz62pU96V6/F3ZDeDBFRa2S1sOjWrRt8fHzw2muv4fjx47h48SISEhJw8uRJxMfHY8iQIdi9ezdWrVqFq1evYu/evZg/fz6efPJJQw+jpfLzcUNFpR5H0nNsXQoRUbOw2gS3VCrF6tWrkZiYiBkzZqCwsBCBgYFITk5GQEAAAgICoNfrsXr1anz44YdQq9WIiYnBzJkzrVVSkwnp/AB8PFyw5UAmevt7QSwW2bokIiKrsuoEd3Np7gluAEg9l4vlKWcwJTYAfQK9G/QctuBok4AA2+woHK3NdjXBbc96dPHAQx4u2HIwk3MXRGT3GBYNJBaJ8OfHOiI7vwTHzuXauhwiIqtiWDRCzy4eeOgBF2w+cJm9CyKyawyLRhCLRIh9rAOy80uQep69CyKyXwyLRnq0qycefMAFmw9kQt/6rxUgIqoTw6KRxCIRYvt2wG+/30Eq5y6IyE4xLJpAr66e+FNbJbawd0FEdoph0QTE4qq5i+u/38GJ83m2LoeIqMkxLJpIWFcv/KmtsurKKPYuiMjOMCyaiFhcNXeRlcfeBRHZH4ZFEwrz94K3u5JXRhGR3WFYNKE/ehfFOJnxu63LISJqMgyLJhYW4AkvjTPnLojIrjAsmpiTWIzYxzrgWm4xTl1g74KI7APDwgp6B3jB827vwg7uAE9ExLBUL2EgAAASA0lEQVSwBiexGLF9O+BqTjFO/creBRG1fgwLK+kT6AVPN2ds2s/eBRG1fgwLK3ESizH0bu/i51/zbV0OEVGjMCysKLybFzzcFNjEuQsiauUYFlZU3bu4cqMIP19k74KIWi+GhZWFB3rjAVcFNnPugohaMYaFlUmcqnoXmTeKkHaJvQsiap0YFs2gb7eq3sWm/ZnsXRBRq2TVsLh06RK6dOlS6+frr78GAKSnp2Ps2LHo3r07IiIisGbNGmuWYzPVvYvL2YVIu3TT1uUQEVlMYs0nP3/+PFQqFXbs2GG0Xa1W4+bNmxg/fjyioqKwYMECnD59GgsWLIBarcaIESOsWZZN9O3mjS0HMrH5wGUEdXKHSCSydUlERGazalhkZGTA19cXHh4etfZ98sknkEqlWLBgASQSCXx9fXHlyhWsWrXKLsNC4iRGTN/2+GzHeZy9fBPdOrW1dUlERGaz6jDU+fPn4evrW+e+1NRUPProo5BI/sir3r1749q1a8jJybFmWTbTL+hPaNtGjpT9l1FQVI7yCh3nMIioVbB6z6J9+/YYNWoUrl69ig4dOmDq1Kno168fcnJy8Mgjjxgd7+npCQDIzs6Gl5eXNUuziareRQd8tuM8Zn94AADgJBbBRSGBs0IKF4UESrkESoUEyhqPnRUSuCikNfZVPXaWO8FJzGsUiMj6rBYWJSUlyMrKgru7O2bPng0XFxds3rwZkydPRnJyMsrKyiCTyYzOqX5cXl5u0Wu1batqcJ0eHuoGn9sQzz7RBR19NMi7VYriEi3ulFaguLTC6L/5eXcMj/X6+/c8nOUSuDhLoXKWGv9XKYVKIYWL8u42hRQqpQx3KgsNxyhkTg4zd9Lc/59bArbZ/jVne60WFkqlEsePH4dUKjWEQLdu3XDx4kWsXr0aCoUCWq3W6Jzqx0ql0qLXys8vNvmmWhcPDzXy8oosPq+x2j+gRPsHTLdREASUV+hQUlZZ9VNeiTtlFbUel5ZV4s7dx9eLyu5ur0S5Vnff53cSi6rCRvFHb0Z593F9vZnqno5SLoHEqXX0amz1/9mW2Gb715j2isUiiz9kW3UYysXFpdY2Pz8/7NmzB+3atUNubq7RvurH3t7e1iyr1RCJRFDIJFDIJHBvY/n5Or0epeU6o4CRyCXIzi0yCpiSu/vvlFXi99tlhsc6EwEslzr9ESTyu2FTz1BazcfOcolD9WqI7IHVwuLkyZOYOHEiPv30UwQHBxu2nzlzBp07d0ZQUBC++OILVFZWGia5Dx8+jA4dOtR59RRZzkkshspZDJWz1LDN3E8jgiBAW6m/GzIVtYKlrp5OfmEZruVWoKS8EqXl9+/ViEWie4LFuGdjqqfTWno1RPbCamHRrVs3+Pj44LXXXsPrr78ONzc3fPnllzh58iQ2bNgALy8vrF69GvPnz8eUKVNw5swZfPLJJ3jjjTesVRJZQCQSQS51glzqBI1abvH5er1QFS4mAqbmUNrNwnLD8ZW6+/dqZFKx0bCYi7z+gHnwdjm0ZVrDUBp7NUSWEwlWvHYzJycHiYmJOHDgAAoLCxEYGIhZs2YhLCwMAJCWloaFCxfi7Nmz8PDwwPjx4zFu3DiLX6e1zVnYUmtps7ZCZ5h7qRoyq7gbJPf2dO6GkeHPlSgtr8T9/jaIRDAaGjPqzcilNXo6VY/v7elIJS2/V9Na/j83JUdrc3PPWVg1LJoLw8J8jtBmvV5AmfbunExZJaQKKbJzCg2PS8qNezYl94RRRaX+vs8vk4jrvgBALjVM/rsoJJDbsAfTpo0ChYVlNnltW3G0NnfwcYOHSmb6wDq0uAluIlsQi0V3J9ur5mo8PNR40E1h9vkVlboaQ2TGvZe6ejq3i7XIzr9jCKBW/+mLWgWJkwjL/j4AcplT87xes7wKUSsilTjBVeUEV1UD5moEAWXlOpSUV5i8dNmaNO4uKLh5x2avbwuO1ub2PhpUllc02+sxLIiakOEqL4Vt/2l5eKihdHKsSXxHa7OmjQJ5ec0XFi1/po6IiGyOYUFERCYxLIiIyCSGBRERmcSwICIikxgWRERkkl1cOisWN/xyucac21qxzY6BbbZ/DW1vQ86zi9t9EBGRdXEYioiITGJYEBGRSQwLIiIyiWFBREQmMSyIiMgkhgUREZnEsCAiIpMYFkREZBLDgoiITHK4sNDr9UhKSkL//v0REhKCiRMn4sqVK7Yuq0kVFxdj0aJFGDRoEEJDQ/HMM89g165dhv1ZWVmIi4tDjx490LdvXyxZsgSVlZU2rLjpXL58GaGhofj6668N2+y5vSkpKRgyZAiCgoIQExOD7du3G/bZY7u1Wi0SExMRERGB0NBQPP/88zhx4oRhvz21eeXKlRg9erTRNnPa98UXXyAyMhLBwcEYOXIkTp8+3TQFCQ4mKSlJ6NOnj7B3714hPT1dmDx5shAZGSmUlZXZurQmM336dCEqKko4cOCAkJmZKaxYsULo2rWrcPDgQaG8vFyIjo4W4uLihPPnzws7d+4UwsLChHfffdfWZTeaVqsVnnnmGcHPz0/YsGGDIAiCXbc3JSVF8Pf3Fz755BMhMzNTWL58udC1a1chNTXVbtv97rvvCuHh4cJPP/0kXL58WXjllVeE0NBQ4caNG3bV5nXr1gldunQRRo0aZdhmTvu+/fZbISQkRNi8ebNw4cIFYe7cuUKvXr2E33//vdE1OVRYlJeXC927dxfWrVtn2FZUVCSEhIQI//nPf2xYWdPJzc0V/Pz8hD179hhtHzdunDBr1ixh8+bNQmBgoHDr1i3Dvg0bNgjdu3cXSktLm7nappWYmCiMHTvWKCzstb16vV4YOHCgsHDhQqPtEydOFD744AO7bXdMTIzw9ttvGx4XFRUJfn5+wtatW+2izTdu3BDi4uKE7t27C0899ZRRWJjTvujoaGHx4sWG/ZWVlcLAgQOFDz74oNG1OdQwVHp6OkpKStCnTx/DNpVKhYCAAKSmptqwsqbj7OyMjz/+GI8++qjRdpFIhNu3byM1NRX+/v5wdXU17OvduzdKSkpw9uzZ5i63yRw7dgzr16/H4sWLjbbba3svXbqE69evY+jQoUbb16xZg2nTptltu93d3bFnzx5kZWVBp9Ph66+/hkwmM/wbbu1tPnv2LFxcXLB582aEhIQY7TPVvt9//x2ZmZno3bu3Yb+TkxN69uzZJO9vDhUWOTk5AAAvLy+j7Z6ensjOzrZFSU1OpVJhwIABUKlUhm2nTp3C4cOHERERgZycHHh7exud4+npCQC4ceNGs9baVAoLCzFnzhy8+uqr+NOf/mS0zx7bCwCZmZkAqsbwp0yZgvDwcDz33HPYvXs3APtt92uvvQaJRILIyEgEBQUhMTERS5cuRYcOHeyizYMGDUJiYiLatWtXa5+p9lW/v9V1TFO8vzlUWJSWlgIAZDKZ0XaZTAatVmuLkqzu4sWLmD59OkJCQjBy5EiUlZXV2X4AKC8vt0WJjbZgwQJ0794dsbGxtfbZY3uBqosYAGDOnDl48sknkZycjH79+mHq1Kk4cOCA3bb7woULUKlU+PDDD7F+/Xo888wzmDNnDs6ePWu3ba5mqn3Wfn+zi8WPzKVQKABUfRqr+QvVarVQKpW2Kstqjh07hunTp+PBBx/EypUrIZVKoVAoav3FqX7cGn8HKSkpSE1NxZYtW+rcb2/trSaVSgEAEyZMwPDhwwEA/v7+OHPmDJKTk+2y3b/99hvmzJmD1atXG4aSg4KC8OuvvyIpKcku21yTqfbVfH+795imaL9D9Syqhyhyc3ONtufm5tYammrtNm/ejAkTJiAwMBCff/453NzcAFR1Uetqf/W+1ubbb79Ffn6+4VLK0NBQAMA///lPxMTE2F17q1XX7ufnZ7S9c+fOyMrKsst2nz59GhUVFQgKCjLaHhISgszMTLtsc02m2vfggw8abat5TFO8vzlUWHTt2hUqlQpHjx41bCsuLsYvv/yCsLAwG1bWtLZs2YI5c+Zg8ODBWLlypdH8Ra9evZCeno7CwkLDtiNHjsDFxQUBAQG2KLdR/u///g/btm1DSkqK4QcApk+fjlWrVtlde6sFBATAxcUFaWlpRtszMjLw8MMP22W7q9/wz58/b7Q9IyMDHTt2tMs212Sqfe7u7ujYsaPR+5tOp8Px48eb5v2t0ddTtTLvvvuuEBYWJvzwww+G71lER0cL5eXlti6tSWRnZwshISHCuHHjhJycHCE3N9fwU1BQIJSVlQlPPPGEMHHiRCE9PV3YtWuX0Lt3byEpKcnWpTeZmpfO2nN7P/zwQyE0NFTYtGmTcOXKFWH58uVCly5dhIMHD9plu3U6nTB69GjhySefFA4dOiRcvnxZWLp0qdC1a1fh+PHjdtfmuXPnGl06a0771q9fLwQFBQnffvutcOHCBeHll18WwsLChPz8/EbX43BhUVlZKSxZskQIDw8XunfvLkyaNEm4evWqrctqMp9++qng5+dX50/1X7zMzExh4sSJQlBQkPDYY48J7777rqDT6WxcedOpGRaCYN/tXbt2rRAZGSkEBgYKsbGxwvfff2/YZ4/tvnXrlrBgwQIhIiJC6N69uzBy5Ejh0KFDhv321OZ7w0IQzGvfmjVrhAEDBgjBwcHC6NGjhTNnzjRJPSJBEITG90+IiMieOdScBRERNQzDgoiITGJYEBGRSQwLIiIyiWFBREQmMSyIiMgkhgU5nJdffhlr1qy57zEbN25EXFxcM1VUJSsry3C7EqKWhmFBREQmOdRdZ8lx6PV6LFq0CD///DPu3LkDQRCQkJCAnj17Gh0XEBCAv/3tb9i3bx9KSkowa9YsREdHAwDy8vIwZcoUZGdnw8nJCYmJifD19cWpU6ewZMkSaLVa5OXloW/fvli0aJHR816+fBmjRo3Cvn37IJPJoNPpEBERgU8++QRFRUUmz1+2bBkKCgrw+uuv13pcVFSEhQsXIiMjAxUVFQgPD8ecOXMgkUiQlJSEH374AVKpFBqNBm+//bZhzQOixmDPguzSzz//jNzcXKxfvx7btm3DsGHD8PHHH9c6TqfTwdnZGRs3bsR7772H+fPn4+bNmwCAa9eu4ZVXXsGWLVvw6KOPGoauPvvsM8yYMQNff/01tm7dit27d+PMmTNGz9uxY0d07tzZsBjR/v374ePjA19fX7POv59FixYhMDAQGzduREpKCgoKCrB27VpkZ2fj008/xbfffouNGzfisccew+nTpxv6KyQywp4F2aXQ0FC4urriq6++wrVr1wx356zLmDFjAFTdldjPzw/Hjh0DAAQHB6N9+/YAqtaK+OGHHwAA77zzDn766Sd89NFHuHTpEsrLy1FSUlLreZ999ln85z//wVNPPYWNGzdixIgR9z2/+jbypuzduxdpaWn45ptvAFQtigNUrQDZtWtXDBs2DAMGDMCAAQMQHh5u7q+M6L4YFmSX9u7di4ULF2LChAmIjIxEp06dsHnz5jqPdXJyMvxZr9cbHkskf/zzEIlEqL6N2pgxY9ClSxf0798fgwcPxs8//4y6brE2ePBgvPPOO7h48SKOHTuGd955x+zza74eAFRUVBjV+P7778PX1xdA1bKyIpEIYrEY69atQ1paGg4dOoRFixahf//+mDNnjkW/O6K6cBiK7NKBAwcwcOBAPP/88+jWrRt27twJnU5X57HVa2CcPXsWly9fRq9evep93sLCQqSlpeGll15CdHQ0bty4gatXr0Kv19c6Vi6XIyYmBi+//DKio6Ph7Oxs9vkajQZnz56FIAgoLi7Gnj17DPv69euHTz75BIIgQKvVIj4+HuvWrcO5c+cwdOhQ+Pr6Ii4uDuPHj6+13gVRQ7FnQXZp1KhRmD17NmJjY1FZWYnHHnsM33//fZ1v6idOnMCGDRug1+uxdOlSuLq61vu8bdq0wZQpUzBs2DAolUp4eXmhR48euHLlSp1DPs899xzWrVuHBQsWmDy/Xbt2hvP+/Oc/Y9++fYiOjoaXlxfCwsIMPY1XXnkFCxcuRGxsLCoqKtC3b19MnjwZUqkUgwcPxvDhww3LbL766quN/E0SVeEtysmhdenSBYcOHYK7u7utSyFq0TgMRUREJrFnQUREJrFnQUREJjEsiIjIJIYFERGZxLAgIiKTGBZERGQSw4KIiEz6fyC3ckML+kJ6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mulnb(train_tfidf,train_y,cv_tfidf,cv_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best roc auc score is 60.771329502755336\n",
      "Confusion Matrix of test set:\n",
      " [ [TN FN]\n",
      " [FP TP] ]\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEFCAYAAADpIfy5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XtcVVXex/EPFxGBgSITvIuZCiqmJJST5Yj6TKVNakJhEyNapmJpqeV1zFsZkpiXoswJy6I0arSZspKhi3mDLprhPTQsUWyQuMuB5w8fz9MJhUMGB/b+vnvt18uz9jprr/0yvyx+e599nCorKysRERHDcXb0BEREpG4o4EVEDEoBLyJiUAp4ERGDUsCLiBiUAl5ExKAU8CIiBqWAFxExKAW8iIhBKeBFRAxKAS8iYlAKeBERg3J15MF9vK5x5OGlAeru097RU5AGatuJ1Mt6/7nco3b3bdK842Udq6FwaMCLiNSbCoujZ1DvFPAiYg6VFY6eQb1TwIuIOVQo4EVEDKlSK3gREYOylDt6BvVOAS8i5qCLrCIiBqUSjYiIQekiq4iIMekiq4iIUWkFLyJiUJZzjp5BvVPAi4g5qEQjImJQKtGIiBiUVvAiIgalFbyIiDFVVugiq4iIMWkFLyJiUKrBi4gYlB42JiJiUFrBi4gYlGrwIiIGpS/8EBExKK3gRUSMqbJSF1lFRIxJK3gREYPSXTQiIgalFbyIiEHpLhoREYNSiUZExKBUohERMSgFvIiIQalEIyJiULrIKiJiUCYs0Tg7egIiIvWissL+rRbKysqIj4+nf//+9OrVi6ioKL744gvr/uzsbMaNG0fv3r3p27cvcXFxlJfb/jaxfv16wsPDCQ4OJjIykj179tjst2eMi1HAi4g5VFTYv9XCqlWreOutt1iwYAFvv/02HTt2ZOzYseTk5FBWVsaYMWNwcnIiOTmZBQsWsHHjRlasWGF9f0pKCnFxcUyePJmUlBQCAgIYO3YsZ86cAbBrjEtRwIuIOdRRwG/dupU77riDfv360aFDBx5//HEKCwvJyMhgy5YtnDhxgiVLltC5c2fCw8OZOnUq69ato6SkBIDExESioqIYOnQonTp1YtGiRXh5eZGcnAxg1xiXooAXEXOorLR/qwVfX1/+85//kJ2djcViYcOGDbi5uREUFER6ejqBgYH4+PhY+4eFhVFUVMS+ffvIzc0lKyuLsLAw634XFxdCQkJIT08HqHGM6ugiq4iYgx016wvy8/PJz8+v0u7t7Y23t7dN25w5c5g8eTLh4eG4uLjg7OxMQkICHTp0ICcnB39/f5v+LVq0AODkyZO4u7sDXLTP3r17AWocozoKeBExh1pcPE1KSmLlypVV2mNjY5k0aZJN26FDh/Dy8mLVqlX4+fmxYcMGpk+fziuvvEJJSQmenp42/d3c3AAoLS2luLjYpu2XfcrKygBqHKM6CngRMYda1Najo6MZNmxYlfZfr95/+OEHpk+fzpo1a7jhhhsA6NGjB4cPH+bZZ5/F3d3dGtQXXHjt4eFhXcFfrI+HhwdAjWNURwEvIuZQi9r6xUoxF7Nnzx7OnTtHjx49bNp79uxJamoqN954I5mZmTb7Tp06BZwvy7Rq1cra1qVLF5s+fn5+1n7VjVEdXWQVEXOog7toLgTsgQMHbNoPHjxIQEAAffr0ITMz06aev3PnTjw9PQkKCsLX15eAgAB27dpl3W+xWMjIyCA0NBSgxjGqo4AXEXOog4APDg4mJCSEmTNnsmPHDrKyskhISODzzz/ngQceYODAgfj5+TFlyhT2799Pamoq8fHxjB492lpHj4mJISkpiZSUFA4fPszs2bMpLCxk5MiRAHaNcSlOlZW1vCfod+TjdY2jDi0NVHef9o6egjRQ206kXtb7i16YYndfjweW2d337NmzJCQkkJaWRl5eHl26dGHy5MnWmvyxY8eYP38+u3fvxtvbmxEjRvDwww/j7Pz/6+u1a9eSlJREXl4e3bp1Y9asWXTr1s26354xLkYBLw2KAl4u5bID/vmH7e7r8eDyyzpWQ6ESTT1wc3NjV8YWVj//NABRo0ZwtuDIJbdfvm/xU7M4eGQH2T9+zbpXV+Lv38JRpyGX6aZBffnwwLtV2sPv+BPrPlpD6pH3Sf5sHXeNrnr3xgUPzZvA00mLqrS7uDgz5tFo3tr1OlsP/5sX311NyE29ftf5N3p19CyahkwBXw8enzGJLl06WV9v2fIfwv80wmaLHHk/JSWlJP3jDWu/ZcsXcM89w5g3N46J4x+je/dANqS8VOOvZdLwdL++G3NXzAAnJ5v28Dv6M2/VLHam7WbqfTNI3ZzGlIWTuHXk4CpjjPjbnUTef9dFx5+8YBJ3PzCSV1a+zoyYuZw+mUvcy4tpd03bOjmfRqmi0v7NIHSbZB0LDg5i3PhocnPPWNvO5P7EmdyfbPqtf/15jh/PZvq0JwAICGjHPVHDGBszhZS3/gXA3r37yfjyQ24fMpDNmz6ov5OQ36yJWxNGjhnO/dNGU1JUgqub7Q/nCbPG8XbSJlYtTATgi21f0rKNP336hfDehvN/x1dcdQUTZj3An+8axM9nC6oco01Aa+7861DmPDiftH99cn6c7V+R9OGLXH9Tb44f+b6Oz7KRMOHjghXwdcjFxYVVzz3Fs8tfZMjQqiuyC8LD+zFk6CCG/+VvlJSc/2TazbfcCMD77/1/3fHokSwyMw8xcNAtCvhG4oY/hfLX2ChWLUzE50pv7h4XYd3XNbgz/m38+Od627LNE5MW27yOfmgUwaHdeWTUY0Q/dG+VY/Qb/Efy8362hjtA+blyRvUf/TufTSNnsTh6BvWuxoAvKyvj/fffJz09nR9//JHS0lI8PDzw9/cnNDSUwYMH4+qqnxMXM+WRcTRxa8IzS5+vNuDnzZ/G1o8+YevWT61tnToFkJNzmqKiYpu+Wd8d55pOHepqyvI7y/z6ACNvjKIgv5CYR6Jt9l0T2BE4vxBYuXEZ3UOC+Cn3v7yy8jXeTtpk7ff2uk2snP8cFkvFRQP+mqCOHDt8nP6338z900bTJqAN3x34joS5K/lqx54q/U3LhCv4aou5x48f5/bbb+fvf/87R44cwcvLi5YtW9KsWTMOHTrErFmzGDp0KNnZ2fU130bj2s4deXTaBCZNnMm5c+cu2e+mfmEE9+zG8oQXbdr/4O1FQUFhlf4FBYV4/8Hrd5+v1I3ck7kU5Ff9e4TzpZfycgtLXl7Iro9388i9j/Pp+9uYungy4Xf0t/Y7fuR7LJZLh9MVvj60DWjNw09M5JWVrzHtvhnknckj/tWn8G/j93ufUuOlGrytJ554goCAAN5++228vKqGSkFBAVOmTGHBggUkJibW2SQbGycnJ1aueopX1m1g964vq+37t9F3s2/fAT5O+7zKGBe7g9XJyYkKA/0PaGauTVxxdXVh06vvsm7Fa8D/1eDb+TN6yn1s3ZRm9zi+V/syYfhkvt55fsW+Z9c3vPn5q4yacDfxM41xy99lM9DdMfaqdgWfkZHB1KlTLxruAF5eXjz66KPs3r27TibXWI0bH03bdq1YvHAZLi4uuLi4AOfD+cKfAVxdXRk0uL/1Iuov5Z/9GS8vzyrtnp4e5Of/XHeTl3pTXHi+/LYjzfbfz+5PMmjbsS2uTewrfRYXFlNcVGwNd4CS4hK+yfiWa7oG/H4TbuxMuIKvNuC9vb3JycmpdoATJ07U+EQzsxkydBCtW7fkWPaX/HT2ID+dPUhwcBBRo4bz09mDtGvXGoDQsF5ccYU3mzdtqTLGkSNZ+Pldjbt7U5v2DgHtOHToaL2ch9St7KwTADT5VZC7urri5ASVdtaMs7N+wNnZBRcX23/Ork1cavvdFYZWWVFh92YU1Qb8XXfdxeOPP05ycjJHjx6lqKiI8vJyioqKyMrK4s0332TWrFkMHz68vubbKEx+aDb9+91psx06eJT3/r2V/v3u5Mcfzz8JLiSkJ2fP/syB/YerjPFx2ue4urpy623h1raO13QgMPDaKuUcaZy+2rGH0uJS/jTkFpv2vgPD2P/1gWrr7r+0++N0mrq78cdBfa1tXt6e9Ajpxt706r/xx1QsFvs3g6j2d8BJkybh5OTE008/bX0w/S95enoyatQoHn7Y/o8Am8HhQ99VaSsuKeGnn/L48su91rbAoM4cOVy1L8B33x3n7ZR/sXzFYry9/0Be3ln+Pm8a33yzn3c3f1hnc5f6U1RQxLqV64l5JJqigiK+3PE14UP/xHU39GTqX2fYPc6uT9LZ/WkGM5ZOxedKb3JzzvDX2HsA2PDSW3U1/cbHQKUXe1Ub8E5OTkyaNIlx48axf/9+cnJyKC4uxt3dHX9/f7p27Vrj08zk0q6++irOnq36tWAXTHjwMZ5cMpsn5k/HydmZj9O2MX3qfCoM9Cuk2b2c8CoF+YXcFTOMex6M5Puj3zPr/nnsTKvdda0ZMXN4cMb9PPBYDM08m/FNxrdMGDGZM6d+qvnNZmHCfzd62Jg0KHrYmFzK5T5srHDu3Xb39ZyffFnHaij0CSURMQcT3iapgBcRc1ANXkTEmCrLjXN3jL0U8CJiDlrBi4gYlGrwIiIGpRW8iIgxVSrgRUQMShdZRUQMSit4ERGDUsCLiBiTA5/K4jAKeBExB63gRUQMSgEvImJMleX6oJOIiDGZL98V8CJiDvqgk4iIUSngRUQMSiUaERFjUolGRMSgKsvNF/DOjp6AiEi9qKjFVkvvvPMOt912Gz169OD222/nvffes+7Lzs5m3Lhx9O7dm759+xIXF0d5ebnN+9evX094eDjBwcFERkayZ88em/32jHExCngRMYXKCvu32vjnP//JzJkziYyM5N1332XIkCE88sgjZGRkUFZWxpgxY3ByciI5OZkFCxawceNGVqxYYX1/SkoKcXFxTJ48mZSUFAICAhg7dixnzpwBsGuMS1HAi4g51MEKvrKykuXLl3PvvfcSHR1N+/btGT9+PH379mXHjh1s2bKFEydOsGTJEjp37kx4eDhTp05l3bp1lJSUAJCYmEhUVBRDhw6lU6dOLFq0CC8vL5KTkwHsGuNSFPAiYgp1sYI/evQoJ06cYMiQITbtL730EhMnTiQ9PZ3AwEB8fHys+8LCwigqKmLfvn3k5uaSlZVFWFiYdb+LiwshISGkp6cD1DhGdXSRVURMobLmkrVVfn4++fn5Vdq9vb3x9va2vs7KygLOl1EeeOAB9u7dS5s2bRg/fjwDBgwgJycHf39/mzFatGgBwMmTJ3F3dwe4aJ+9e/cC1DhGdRTwImIKtVmZJyUlsXLlyirtsbGxTJo0yfq6oKAAgOnTpzNx4kSmTJnCBx98wIQJE3jppZcoKSnB09PTZgw3NzcASktLKS4utmn7ZZ+ysjKAGseojgJeREyhNgEfHR3NsGHDqrT/cvUO0KRJEwBGjx7NiBEjAAgMDOSbb75h7dq1uLu7W4P6gguvPTw8rCv4i/Xx8PAAqHGM6qgGLyLmUOlk9+bt7U2bNm2qbL8O+Aulk86dO9u0X3vttWRnZ+Pv78+pU6ds9l147e/vT6tWrWzaftnHz8/P2q+6MaqjgBcRU6iLi6xBQUF4enpa6+UXHDx4kHbt2tGnTx8yMzNt6vk7d+7E09OToKAgfH19CQgIYNeuXdb9FouFjIwMQkNDAWocozoKeBExhcoKJ7s3e7m7uzN27FhWr17Npk2bOH78OM899xyfffYZMTExDBw4ED8/P6ZMmcL+/ftJTU0lPj6e0aNHW+voMTExJCUlkZKSwuHDh5k9ezaFhYWMHDkSwK4xLkU1eBExhQqL/cFdGxMmTMDDw4Nnn32WkydP0rFjR1asWMGNN94IwJo1a5g/fz4RERF4e3sTGRnJxIkTre+PiIigoKCA5cuXk5eXR7du3Vi7di2+vr4ANG3atMYxLsWp0oHfROvjdY2jDi0NVHef9o6egjRQ206kXtb7s8MG2N23zc7LO1ZDoRW8iJhCbUovRqGAFxFTcFytwnEU8CJiClrBi4gYVF1dZG3IFPAiYgpawYuIGFRlpQJeRMSQavtFHkaggBcRU6jQCl5ExJhUohERMSjdRSMiYlC6i0ZExKBUgxcRMSjV4EVEDErPohERMSiVaEREDKpCF1nrV2FZiSMPLw1Q2tdrHD0FMSit4EVEDEoXWUVEDEoreBERgzLhTTQKeBExB0uFs6OnUO8U8CJiCiZ8WrACXkTMoRLV4EVEDKnChEV4BbyImEKFVvAiIsakEo2IiEFZFPAiIsaku2hERAxKAS8iYlCqwYuIGJQJnxasgBcRc9BtkiIiBmVx9AQcwHxP3xERU6pwcrJ7+62+++47evXqxYYNG6xt2dnZjBs3jt69e9O3b1/i4uIoLy+3ed/69esJDw8nODiYyMhI9uzZY7PfnjEuRgEvIqZQWYvttzh37hxTp06lqKjI2lZWVsaYMWNwcnIiOTmZBQsWsHHjRlasWGHtk5KSQlxcHJMnTyYlJYWAgADGjh3LmTNn7B7jUhTwImIKFbXYfosVK1bg6elp07ZlyxZOnDjBkiVL6Ny5M+Hh4UydOpV169ZRUnL+K0sTExOJiopi6NChdOrUiUWLFuHl5UVycrLdY1yKAl5ETKHCyf6ttnbv3s0bb7zBkiVLbNrT09MJDAzEx8fH2hYWFkZRURH79u0jNzeXrKwswsLCrPtdXFwICQkhPT3drjGqo4usImIKtXlUQX5+Pvn5+VXavb298fb2rtJ3+vTpzJ49m5YtW9rsy8nJwd/f36atRYsWAJw8eRJ3d3eAi/bZu3evXWNURwEvIqZQm5V5UlISK1eurNIeGxvLpEmTbNrmzZvHddddx9ChQ6v0LykpqVK2cXNzA6C0tJTi4mKbtl/2KSsrs2uM6ijgRcQUalNbj46OZtiwYVXaf716f+edd0hPT2fz5s0XHcfd3d0a1BdceO3h4WFdwV+sj4eHh11jVEcBLyKmUJu7Yy5WirmYt956izNnztC/f3+b9vnz5/Pyyy/Tp08fMjMzbfadOnUKOF+WadWqlbWtS5cuNn38/Pys/aobozoKeBExhbp4VMHSpUur3MkyePBgYmNjGTJkCF999RUpKSnk5+dbf2Ds3LkTT09PgoKCcHNzIyAggF27dtGvXz8ALBYLGRkZREZGAtCnT59qx6iO7qIREVOoi9sk/fz8aN++vc0G4OvrS+vWrRk4cCB+fn5MmTKF/fv3k5qaSnx8PKNHj7bW0WNiYkhKSiIlJYXDhw8ze/ZsCgsLGTlyJIBdY1yKVvAiYgoWBzyKpmnTpqxZs4b58+cTERGBt7c3kZGRTJw40donIiKCgoICli9fTl5eHt26dWPt2rX4+vraPcalOFVWVjrsq2hd3Vo76tDSQBX/8KmjpyANVJPmHS/r/avb3mt33wnfv3pZx2ootIIXEVPQF36IiBiUw0oVDqSAFxFT0Bd+iIgYlEo0IiIGZcYv/FDAi4gpqEQjImJQKtGIiBiU7qIRETGoChNGvAJeRExBF1lFRAxKNXgREYPSXTQiIgalGryIiEGZL94V8CJiEqrBi4gYlMWEa3gFvIiYglbwIiIGpYusIiIGZb54B2dHT8AsPD09WJ6wkBPff8V/zxzg3++uJzg4yLrf3d2dJxfP5MihnZw5ncmHW97kuuu6OXDGcrmKiopZFL+am4fcQ+jA4TwwZRb7Dx217i8pLeWZ1S8xaHg0NwweQcykx8k8ePiS4y1ZnsiEaX+v0l5ebmHlmlcYOPw+rh9wJ3ePfZgd6V/WyTk1ZhW12IxCAV9PNrzxItH3RRD/zHNERN7PyZzTpKWm0LnzNQDEL53H+Af/xtL41dwT9SAWi4UPt7xJ69YtHTxz+a0mz1rIO+99yOioESxbNIvmvlcSPWEq3x3LBmDJ8hd4PeVdYkbdRfyCmTg7OzPmoRmcPHW6ylivbdzEK2++c9HjPJnwHOuS32bsXyNY/uQcWlx9FROnzePose/r9PwaGwuVdm9GoRJNPejdqweDB/dn/ITHeHHN+W9r//CjT7i20z95Yt40okaNZ1TUcBKWv8BzzycB8Pn2dE7+sIfIiDt4ZlmiI6cvv8G+/Yf4fNcXzJ02iYg7bwPgj2EhjBr3CCvXrCPuicd594NUoiOHcc+IoQBc1z2QfkPu5r2PPmZ01F0AnPlvHstWr2XT+1v5g5dnleMc+/4Eb77zb+IXzGDwn/oBENo7mOHRE9mZ/hUd27etpzNu+MxYg9cKvh5c27kjAB98mGbT/vnnuxk86BacnZ1xc2tCfv7P1n2FhUWUlpbh63tlfU5VfifHvj8BnA/1X+rVI4htOzOoqKjk3LlyPD09rPuaNXPHrUkTzuYXWNteTErmiz37SHxmIV2u7VjlOKmfbsfH+w8M6n+Tta1JkyZsfu0F6w8OOa+yFptRKODrQfb3PwDQrm1rm/YOAe3w8fHG2/sPvPDiq0ycEMP1IT254gofnlo8i2bN3ElJ+ZcjpiyXyb9FcwB+zDll037ix5MUFBZRWFRExF9u5bWNm9mbeYCz+T/zzOq1lJSWMaj/H639I4cNYdP6F7ixT6+LHufgkSwC2rfhw7TPGBr1AD1vvp0R0RPZ/eWeuju5RqqCSrs3o1CJph7sTv+aAwePsGLFYsaMmcLhI1lEjLyDW/88ADh/AXbBwmcIC+vNju3/BqCiooLRYybzxZd7HTl1+Y26B3amQ9vWLFy6ioWzHqFdm1a8v/UTPt2eDkBRcQnjY0bx9b793DN2MgBOTk4smv0o3bpeax0noH2bao/z37yzHPv+B5YsT+ShcX+jue+V/OO1txj/6Fz+uT6R1i396u4kGxkjXTy1V40BHxUVhZOTfY9hW79+/WVPyIjKysoYGTGWV9atYueO9wDYvj2dpfGrmTvnUSoqKvj0k000dXMjevRD/HDiJMOG3caLiUvJz/+ZzZs/cPAZSG25ubmRsHgO059Ywt1jHwagZ/dARo+6i+fWrsfZ2Yl7H3yUsrJzLJ4zFb+rr+LDtG3MfTIBL08PBvS70a7jlJeX89N/83h51dNcf10PAHoFd+PWiBjWrt/AnKmxdXaOjU2lgVbm9qox4G+55RYSEhLo2LEjwcHB9TEnQ/r224OEXD+INm1a4erqQlbW98yZPQWLxUL//n3pfG1HbrjxNtIzvgbgP2nbuOqqK1m+bKECvpHq1LE9KUmr+THnNBaLhTat/Fm9dj3Ozs7s+mIPx74/wetrEugR2AWAsJDryDubz+Jlz9kd8B7NmtHMvSkhPbv/os2dnt27cuhIVl2cVqNlpLtj7FVjwI8bNw4vLy/i4+NJTEykTZvqf2WUqpo1c2f48NtJTf2M7OwfrO09egTyzb4DtG3TivLycmu4X7Bt2y4iI/6Cp6cHhYVF9T1tuQzFJSV8mLaNG0Kuo6Xf1db2g4e/o1PH9pzMOY2LizPdu3a2eV/v4G68v/UTioqK8fBoVuNx2rZpyWc7K7BYKnB1dbG2l5db7P7N2yzMWKKx6yLrqFGjCA0NJSEhoa7nY0jnzpWzeuVTREb8xdrWoUNbbv3zAP71rw85eOgorq6uhIX2tnlfaGhvTp3KVbg3Qq6uriyIW8F7Wz+2tmX/cJJPt++m/x/D6NC2NRZLBXv27bd5355vD+B7hQ/NmrnbdZy+fXpTVnaOtG07rG35Pxfw1d5vua5H4O9zMgZRUVlp92YUdl9knT9/Pvv27avLuRhWeXk5L619jRmPP8Sp07nk5//Mk4tncfr0TyQsf5H8/J/58qtveP2155k772l+/CGH228fyL2jRvDQw7McPX35DZq4ujJ86J95ISkZ3yuvwMvDg2eeW8uVV/pwX+QwPD096HptRx6d+yST7r+PFs2vIm3bTt7dksrMKePtXn33De3NDddfx9wnE8g7+zMtmvuy5pU3Abh35J11eYqNjnFi235OlZWO+3Hl6ta65k4G4e7uzqKFM4gYORR396b8J20bjz2+kO++Ow6Ar++VPPXkLIYOGUyzZu5k7j9E3NLVprtNsviHTx09hd9NSWkpCc/9g/dTP6Gs7ByhvXvy6MQxtP2/Tyfnnc0nfvVLpH22g5LSMjq2b8uYe0daP7D0a3+LnY5Hs2asjnvCpr2oqJiE5//B+6mfUlxcTM/ugTz28Diu7dihrk+xXjVpXvVzALUR1X6Y3X1fO/b2ZR2roVDAS4NipICX39flBvw97e3/jeb1Yxd/LERjo/vgRcQUyk1YpNEnWUXEFCpr8V9tFBQUsHjxYgYMGECvXr0YPnw4W7dute7Pzs5m3Lhx9O7dm759+xIXF0d5ebnNGOvXryc8PJzg4GAiIyPZs8f2k8j2jHExCngRMYW6elzwjBkzSEtLY+HChbzzzjsMHjyY2NhYtm/fTllZGWPGjMHJyYnk5GQWLFjAxo0bWbFihfX9KSkpxMXFMXnyZFJSUggICGDs2LGcOXMGwK4xLkU1eGlQVIOXS7ncGvywdvY/fO3t45vt6nf69GluuukmEhMT6d+/v7U9Ojqa5s2b079/f2bMmMG2bdvw8fEBYMOGDSxevJjt27fj7u7O//zP/xAeHs706dMBsFgsDBo0iBEjRjBx4kQ2b95c4xiXohW8iJhCXTxsrFmzZrz44otcf/31Nu1OTk6cPXuW9PR0AgMDrcEMEBYWRlFREfv27SM3N5esrCzCwsKs+11cXAgJCSE9/fxzi2oaozq6yCoiplCbRxXk5+eTn59fpd3b2xtvb2/ray8vL26++WabPl999RU7duxg9uzZfPbZZ/j7+9vsb9GiBQAnT560rr4v1mfv3vMPGszJyal2jOoo4EXEFGqzMk9KSmLlypVV2mNjY5k0adIl33fkyBFiY2Pp2bMnkZGRfPTRR3h62n5Ri5ubGwClpaUUFxfbtP2yT1lZGQAlJSXVjlEdBbyImEJtLjdGR0czbFjVD0b9cvX+a7t37yY2NpZWrVqRmJhIkyZNcHd3twb1BRdee3h4WFfwF+vj4XH+y2BqGqM6CngRMYXa3B3z61JMTTZt2sTMmTMJDQ3l2WefxcvLCzhfesnMzLTpe+rUKeu+Vq1aWdu6dOli08fPz8+uMaqji6wiYgrV+x9dAAAEDElEQVR1dR/85s2bmT59OrfeeiuJiYnWcAfo06cPmZmZNvX8nTt34unpSVBQEL6+vgQEBLBr1y7rfovFQkZGBqGhoXaNUR0FvIiYQl3cRXPy5EnmzJlDWFgY06ZNIy8vj9OnT3P69Gny8vIYOHAgfn5+TJkyhf3795Oamkp8fDyjR4+21tFjYmJISkoiJSWFw4cPM3v2bAoLCxk5ciSAXWNciko0ImIKlsrf/4nwH3zwAcXFxezYsYN+/WwfEte7d29ef/111qxZw/z584mIiMDb25vIyEgmTpxo7RcREUFBQQHLly8nLy+Pbt26sXbtWnx9fQFo2rRpjWNcij7oJA2KPugkl3K5H3Tq32ag3X3Tsj+6rGM1FFrBi4gpGOmLPOylgBcRUzBfvCvgRcQkanPx1CgU8CJiCgp4ERGDqou7aBo6BbyImEJtP8BkBAp4ETEFB94R7jAKeBExBdXgRUQMSit4ERGDstT621YbPwW8iJiCPskqImJQuotGRMSgtIIXETEoreBFRAxKK3gREYPSowpERAxKJRoREYOq1ApeRMSY9KgCERGD0qMKREQMSit4ERGDslSoBi8iYki6i0ZExKBUgxcRMSjV4EVEDEoreBERg9JFVhERg1KJRkTEoFSiERExKD0uWETEoHQfvIiIQWkFLyJiUBV6XLCIiDHpIquIiEGZMeCdKs141iIiJuDs6AmIiEjdUMCLiBiUAl5ExKAU8CIiBqWAFxExKAW8iIhBKeBFRAxKAS8iYlAKeBERg1LAO1BFRQXPPvss/fr1o2fPnsTExHDs2DFHT0sakMTERO655x5HT0MaKQW8A61atYrXX3+dhQsX8sYbb+Di4sKYMWMoLS119NSkAVi/fj3Lli1z9DSkEVPAO0hZWRlr164lNjaWW265ha5du7Js2TJyc3N57733HD09caCcnBwefPBBli5dSkBAgKOnI42YAt5BMjMzKSoq4oYbbrC2eXl5ERQURHp6ugNnJo62b98+PD092bRpEz179nT0dKQR0+OCHSQnJwcAPz8/m/YWLVrw448/OmJK0kAMGDCAAQMGOHoaYgBawTtIcXExAG5ubjbtbm5ulJWVOWJKImIwCngHcXd3B6gS5mVlZXh4eDhiSiJiMAp4B2nZsiUAp06dsmk/depUlbKNiMhvoYB3kK5du+Ll5cWuXbusbQUFBXz77beEhoY6cGYiYhS6yOogbm5u3HvvvSxbtozmzZvTpk0b4uPj8fPzY/DgwY6enogYgALegR566CEsFgtz586luLiYkJAQ1qxZU+XCq4jIb6Ev3RYRMSjV4EVEDEoBLyJiUAp4ERGDUsCLiBiUAl5ExKAU8CIiBqWAFxExKAW8iIhBKeBFRAzqfwFVY9KtThMweQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "best_alpha=0.01\n",
    "testmulnb(train_tfidf,train_y,test_tfidf,test_y,best_alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tPositive\t\t\t\t\tNegative\n",
      "________________________________________________________________________________________________\n",
      "\t-21.2356\taachen         \t\t\t\t-5.0522\tgreater        \n",
      "\t-21.2356\taarrgh         \t\t\t\t-5.0695\tlovee          \n",
      "\t-21.2356\tabbey          \t\t\t\t-5.1290\ttasteand       \n",
      "\t-21.2356\tabbrevi        \t\t\t\t-5.1415\tgoodad         \n",
      "\t-21.2356\tabdomen        \t\t\t\t-5.1528\tlikeabl        \n",
      "\t-21.2356\tabhor          \t\t\t\t-5.1561\tteabal         \n",
      "\t-21.2356\tabrevi         \t\t\t\t-5.2124\tflavorc        \n",
      "\t-21.2356\tabsinth        \t\t\t\t-5.2350\tcoffe          \n",
      "\t-21.2356\tabsolutelt     \t\t\t\t-5.3152\tproducto       \n",
      "\t-21.2356\tabstin         \t\t\t\t-5.3283\tusei           \n",
      "\t-21.2356\tacceptal       \t\t\t\t-5.3996\tonec           \n",
      "\t-21.2356\taccourd        \t\t\t\t-5.4874\ttriangl        \n",
      "\t-21.2356\taccpet         \t\t\t\t-5.5397\tmakeov         \n",
      "\t-21.2356\tacct           \t\t\t\t-5.5943\tgetaway        \n",
      "\t-21.2356\tacesuflamek    \t\t\t\t-5.6474\tbest           \n",
      "\t-21.2356\tack            \t\t\t\t-5.6475\tpricei         \n",
      "\t-21.2356\tacknowleg      \t\t\t\t-5.6590\tbuy            \n",
      "\t-21.2356\tacquisit       \t\t\t\t-5.6894\tamazon         \n",
      "\t-21.2356\tactualla       \t\t\t\t-5.6973\tfoodbank       \n",
      "\t-21.2356\taddicitv       \t\t\t\t-5.6995\tfindabl        \n"
     ]
    }
   ],
   "source": [
    "def show_most_informative_features(vectorizer, clf, n=20):\n",
    "    feature_names = uni_gram.get_feature_names()\n",
    "    \n",
    "    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))\n",
    "    top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])\n",
    "    print(\"\\t\\t\\tPositive\\t\\t\\t\\t\\tNegative\")\n",
    "    print(\"________________________________________________________________________________________________\")\n",
    "    for (coef_1, fn_1), (coef_2, fn_2) in top:\n",
    "        print(\"\\t%.4f\\t%-15s\\t\\t\\t\\t%.4f\\t%-15s\" % (coef_1, fn_1, coef_2, fn_2))\n",
    "        \n",
    "show_most_informative_features(train_tfidf,mulbnb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#neg_class_prob_sorted = bnb.feature_log_prob_[0, :].argsort()\n",
    "#pos_class_prob_sorted = bnb.feature_log_prob_[1, :].argsort()\n",
    "\n",
    "#print(np.take(uni_gram.get_feature_names(), neg_class_prob_sorted[:10]))\n",
    "#print(np.take(uni_gram.get_feature_names(), pos_class_prob_sorted[:10]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}