NLP 语言情感分析.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 多线程请求"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tomorrow import threads\n",
    "from urllib import request\n",
    "import time\n",
    "import pandas as pd\n",
    "import queue\n",
    "from tqdm import tqdm\n",
    "from retrying import retry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 整数转 IP\n",
    "def int2ip(x):\n",
    "    ch2 = lambda x: '.'.join([str(x//(256**i)%256) for i in range(3,-1,-1)])\n",
    "    return ch2(x)  \n",
    "\n",
    "\n",
    "q = queue.Queue()\n",
    "@threads(100)\n",
    "@retry\n",
    "def int2ip2location(ipi):\n",
    "    ip = int2ip(ipi)\n",
    "    url = \"http://ip.taobao.com/service/getIpInfo2.php?ip=\" + str(ip)\n",
    "    try:\n",
    "        rep = request.urlopen(url)\n",
    "    except:\n",
    "        print(ip,\"cuowu\")\n",
    "    data = json.loads(rep.read())[\"data\"]\n",
    "    q.put(str(ipi) + \",\" + ip + \",\"+data[\"country\"] + \",\" + data[\"region\"] + \",\" + data[\"city\"] + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"/Users/weduoo/Desktop/hls_comm.csv\")\n",
    "len(data[\"ip\"][:300])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "ls = data[\"ip\"][:2000]\n",
    "for ip in ls:\n",
    "    int2ip2location(ip)\n",
    "with open(\"/Users/weduoo/Desktop/iptxt.txt\", 'a') as txt:\n",
    "    while not (q.qsize()==len(ls)):\n",
    "        pass\n",
    "    while not q.empty():\n",
    "        txt.write(q.get())\n",
    "        \n",
    "print(time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "['内蒙古', '内蒙古', '内蒙古', '内蒙古', '内蒙古', '山西', '山西', '山西']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 参考\n",
    "- [使用python+机器学习方法进行情感分析(详细步骤)](http://www.raincent.com/content-10-10586-1.html)\n",
    "\n",
    "## 常见 NLP 库使用\n",
    "### snowNLP\n",
    "- [我用Python爬了鹿晗、关晓彤微博的热门评论，并进行了情感分析](http://www.chinaz.com/program/2017/1011/815429_3.shtml)\n",
    "- [使用snownlp进行评论情感分析](http://jeremy-xu.oschina.io/2017/03/16/%E4%BD%BF%E7%94%A8snownlp%E8%BF%9B%E8%A1%8C%E8%AF%84%E8%AE%BA%E6%83%85%E6%84%9F%E5%88%86%E6%9E%90/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n"
     ]
    }
   ],
   "source": [
    "print(q.qsize())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}