Model Validator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd #dataframes e series\n",
    "import matplotlib.pyplot as plt #graphics\n",
    "import numpy as np #matrixes\n",
    "import nltk #Tokenize\n",
    "import sklearn as skl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn import svm\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer #tf-idf; tf\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Data*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Trainingtable = pd.read_csv('C:/Users/sergiojesus/Desktop/Recursos/comments_toxicos/Dados/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "col_labels = ['toxic','severe_toxic','obscene','threat','insult','identity_hate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TFIDF = TfidfVectorizer(max_df= 0.8 , min_df= 2, tokenizer = nltk.word_tokenize, ngram_range= (1,3))\n",
    "TF = CountVectorizer(max_df= 0.8 , min_df= 2, tokenizer = nltk.word_tokenize, ngram_range= (1,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = list()\n",
    "X.append(TFIDF.fit_transform(Trainingtable.comment_text))\n",
    "X.append(TF.fit_transform(Trainingtable.comment_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "xDict = {'TF Training':1,'TF-IDF Training':0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y = list()\n",
    "for label in col_labels:\n",
    "    Y.append(Trainingtable[label])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Models*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NB = MultinomialNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SVM = skl.svm.SVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LOG = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TREE = DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Feature selection*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def featureselect(Model, X, Y):\n",
    "    Model.fit(X, Y)\n",
    "    print('Model Fited')\n",
    "    featureselectmodel = SelectFromModel(Model, prefit = True)\n",
    "    NewX = featureselectmodel.transform(X)\n",
    "    return NewX"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Validation*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cross_validation_score(Model,X,Y, cores = -1):\n",
    "    return skl.model_selection.cross_val_score(Model, X, Y, cv = 10, n_jobs = cores, scoring= 'roc_auc').mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Test Functions*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Test(Model, X, Y):\n",
    "    Score = list()\n",
    "    for ind, column in enumerate(col_labels):\n",
    "        Score.append(cross_validation_score(Model, X, Y[ind]))\n",
    "        print('Score Created')\n",
    "    print(sum(Score)/len(Score))\n",
    "    return Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Test_With_Feat_Select(Model, X, Y):\n",
    "    Score = list()\n",
    "    for ind, column in enumerate(col_labels):\n",
    "        InsideX = featureselect(Model, X, Y[ind])\n",
    "        print('Features Selected')\n",
    "        Score.append(cross_validation_score(Model, InsideX, Y[ind]))\n",
    "        print('Score Created')\n",
    "    print(sum(Score)/len(Score))\n",
    "    return Score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Tests*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score Created\n",
      "Score Created\n",
      "Score Created\n",
      "Score Created\n",
      "Score Created\n",
      "Score Created\n",
      "0.8558223005535951\n",
      "[0.8978534016923321, 0.8884490778405831, 0.8877716679474622, 0.7869730041864255, 0.8858565023117, 0.7880301493430674]\n",
      "Model Fited\n",
      "Features Selected\n",
      "Score Created\n",
      "Model Fited\n",
      "Features Selected\n",
      "Score Created\n",
      "Model Fited\n",
      "Features Selected\n",
      "Score Created\n",
      "Model Fited\n",
      "Features Selected\n",
      "Score Created\n",
      "Model Fited\n",
      "Features Selected\n",
      "Score Created\n",
      "Model Fited\n",
      "Features Selected\n",
      "Score Created\n",
      "0.8623573617454307\n",
      "[0.6758284355841054, 0.9502962419938313, 0.808006090749493, 0.9716109429577454, 0.830833726436054, 0.9375687327513544]\n"
     ]
    }
   ],
   "source": [
    "print(Test(NB,X[1],Y))\n",
    "print(Test_With_Feat_Select(NB,X[1],Y))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}