Text_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://nlp.stanford.edu/IR-book/html/htmledition/naive-bayes-text-classification-1.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "import sqlite3\n",
    "import re\n",
    "\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read train data\n",
    "# Data in format: text, category\n",
    "# category: 0 - male; 1 - female\n",
    "#data = readCSV(\"train.csv\")\n",
    "data1 = readCSV(\"data.csv\")\n",
    "#print data[0]\n",
    "#print data1[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.934180\n"
     ]
    }
   ],
   "source": [
    "# probabilistic learning method\n",
    "nb = NaiveBayes()\n",
    "nb.trainModel(data1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2016-02-14 23:05:00 ЯНА18ЛЕТ МОИ ИНТИМ ФОТОЧКИ ВЫШЛЮ ПОСЛЕ ПЕРЕВОДА10ГРН 0956954237\n",
      "2016-02-14 23:05:00 ЯНА18ЛЕТ МОИ ИНТИМ ФОТОЧКИ ВЫШЛЮ ПОСЛЕ ПЕРЕВОДА10ГРН 0956954237\n",
      "2016-02-14 23:03:00 всех с праздничком*настроение паршивое((пообщаюсь с адекватным парнем 0969046095 Днепр. Дев 26 лет. Всем любви))*\n",
      "2016-02-14 23:03:00 всех с праздничком*настроение паршивое((пообщаюсь с адекватным парнем 0969046095 Днепр. Дев 26 лет. Всем любви))*\n",
      "2016-02-14 22:47:00 ПЫШН. Ж49 ИЩУ МУЖА 49-54 С ЖИЛЬЕМ в ДНЕПРЕ (Коммунар). 098 49 11 780\n",
      "2016-02-14 22:47:00 ПЫШН. Ж49 ИЩУ МУЖА 49-54 С ЖИЛЬЕМ в ДНЕПРЕ (Коммунар). 098 49 11 780\n",
      "2016-02-14 01:30:00 Молодая многодетная мать  бабушка и прабабушка ищет будущего мужа. На долгие годы. Мне 39. Симпатичная. Днеп\n",
      "2016-02-14 01:28:00 Д 23 ишит прана с/о зват наста мой номир 0951167943\n",
      "2016-02-14 01:26:00 ЯНА18ЛЕТ МОИ ИНТИМ ФОТОЧКИ ВЫШЛЮ ПОСЛЕ ПЕРЕВОДА10ГРН 0956954237\n",
      "2016-02-14 01:23:00 полная девушка ищет парня с млс нет!0638054744 смс!\n",
      "2016-02-14 01:22:00 Молодая пробабушка и многодетная мать  ищет будущего мужа. На долгие годы. Возраст значения не имеет. Днепропетровск. 0994288170\n",
      "2016-02-14 01:22:00 Хачу общяца 80975930532\n",
      "2016-02-14 01:10:00 ДЕВУШКА ПОЗНАКОМИТСЯ С ПАРНЕМ ДЛЯ ОБЩЕНИЯ -0973224297\n",
      "2016-02-14 01:07:00 Неужели больше не осталось хороших девушек? мне29.Зп. 0632248076\n",
      "2016-02-14 01:07:00 Кому не спится ночью - полуночный собеседник! Не вирт. Жду! 0991748090\n",
      "2016-02-14 01:04:00 Д 23 ишит прана с/о зват наста мой номир 0951167943\n",
      "2016-02-14 01:00:00 ДЕВУШКА ПОЗНАКОМИТСЯ С ПАРНЕМ ДЛЯ ОБЩЕНИЯ -0973224297\n",
      "2016-02-14 00:54:00 Девушка 27летДзержинск.Познакомлюсь с парнем для с/о.0999411265\n",
      "2016-02-14 00:49:00 в/жен пообщается по смс с высоким умным парнем с Днепра063 63 62 605\n",
      "2016-02-14 00:38:00 в/жен пообщается по смс с высоким умным парнем с Днепра063 63 62 605\n",
      "2016-02-14 00:38:00 СимпСерьезнП36позн.сСимпД18-32дляС/О совм.жизни.ВозмФлирт.0637167196\n",
      "2016-02-14 00:35:00 Неужели больше не осталось хороших девушек? мне29.Зп. 0632248076\n",
      "2016-02-14 00:34:00 интересная одинокая девушка познакомиться с интересным муж от30и выше для сер отнош 0951264507\n"
     ]
    }
   ],
   "source": [
    "# Read all text from db\n",
    "conn = sqlite3.connect(\"clubnika.db\")\n",
    "cur = conn.cursor()\n",
    "#rows = cur.execute(\"select msg,timestamp from messages  where timestamp between  '2016-02-01' and '2016-03-01'\").fetchall()\n",
    "rows = cur.execute(\"select msg,timestamp from messages  order by timestamp desc\").fetchall()\n",
    "data = [entry[0] for entry in rows]\n",
    "date = [entry[1] for entry in rows]\n",
    "response = nb.classify(data)\n",
    "for i,entry in enumerate(response):\n",
    "    if entry[1]==\"1\": print \"%s %s\" % (date[i], entry[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.000000\n",
      "{'1': -7.906681345001262, '0': -7.10769031284391}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('Chinese Chinese Chinese Tokyo Japan', '0')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Test\n",
    "data = [[\"Chinese Beijing Chinese\",\"0\"],\n",
    "        [\"Chinese Chinese Shanghai\",\"0\"], \n",
    "        [\"Chinese Macao\",\"0\"],\n",
    "        [\"Tokyo Japan Chinese\",\"1\"]]\n",
    "nb = NaiveBayes()\n",
    "nb.trainModel(data, 1.)\n",
    "nb.classify([\"Chinese Chinese Chinese Tokyo Japan\"], test=True)\n",
    "# Must return\n",
    "# pobability {'1': 0.00013548070246744226, '0': 0.00030121377997263036}\n",
    "# or log     {'1': -7.906681345001262, '0': -7.10769031284391}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('One of a kind Money maker  Try it for free From nobody Wed Mar                  Content Type  text html   charset  iso         Content Transfer Encoding   bit   body lang EN US    div class Section      p class MsoBodyText style  text align justify   b CONSANTLY  b  being bombarded by so called  FREE  money making systems that teases you with limited information  and when it s all said and done  blind sides you by demanding your money credit card information upfront in some slick way  b  after the fact  b   Yes  I too was as skeptical about such offers and the Internet in general with all its hype  as you probably are  Fortunate for me  my main business slowed down   i I have been self employed all my life  i    so I looked for something to fit my lifestyle and some other way to assist me in paying my bills  without working myself to death or loosing more money  then  this proposal to try something new without any upfront investment   i great  because I had none  i   interested me to click on the link provided  And I don t regret at all that I did  I am very happy  and happy enough to recommend it to you as a system that is true to its word  I mean absolutely no upfront money  You join only if   i when  i   you make money  You also get to track the results of your time and efforts instantly and updated daily  I especially liked this idea of personal control with real time  staying informed statistics   p    p class MsoBodyText style  text align justify   b This system is quite simply the most logical  opened  and fair of any others that I ve seen before  Why  Because from the start  you get all the specific facts you need to seriously consider if this is right for you   No teasing  No grand testimonies  No kidding  Just the facts  Unlike in other programs that give you  no idea  of their overall plan before first forking over your money credit card  or worst yet  joining and finding out too late  after wasting valuable time trying to figure them out  this system is straightforward and informative  providing you with the two things you really must know    u What s it all about  u    and   u How does it work  u     These are the ultimate deal makers or deal breakers that need to be immediately disclosed  well before discovering that maybe you don t want to do that  by then you are  hooked  and now locked into a frustrating battle to try to get your money back    b   p    p class MsoBodyText style  text align justify  I call this my  Platinum Choice  because it stands alone as a true  superior deal that is totally different from previously misleading   hook first  programs that promise lofty mega money jackpots  but really just want your money upfront to line their own pockets  You ve seen the headlines    u Join free and Make         every week for life  u   yeah  right   p    p class MsoBodyText style  text align justify  I did not make millions yet  but the whole thing was launched just a few weeks ago and I am more than happy with my earnings  so far  I must tell you  I wouldn t be able to do anything without corporate help   which was unusually thorough  timely  and motivating    p    p class MsoBodyText style  text align justify  You have to see this in action for yourself and make up your own mind  just go to my site and fill out the form as soon as you can  You will get your own site in a few minutes  Then you are ready to try whether you can make some decent money with this system and the Internet s explosive potential   fully loaded with hi tech software  free corporate help  on time member s support and even protective safeguards    p    p class MsoBodyText style  text align justify  Get it now  and you can call me at any time with questions  It really could help you like it is helping me to finally be able to pay my bills  and keep my free time free   Good luck   p    p class MsoBodyText style  text align justify   a href  http   www mindupmerchants com default asp ID       http   www mindupmerchants com default asp ID       a   p    p class MsoBodyText style  text align justify  Ben Green                   p    p class MsoBodyText P S Free POP  email is ofered for members now   p    div     body  \\n', u'1')]\n"
     ]
    }
   ],
   "source": [
    "fin = open(\"test.csv\")\n",
    "data = fin.readline()\n",
    "print nb.classify([data])\n",
    "\n",
    "fin.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}