predict.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "051ced2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: LC_ALL=C\n"
     ]
    }
   ],
   "source": [
    "%env LC_ALL=C\n",
    "from cgitb import text\n",
    "import spacy\n",
    "import os\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from spacy.tokens import DocBin\n",
    "from multiprocessing import Process, freeze_support, set_start_method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "573885e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_word_vectors(model_name, word_vectors):\n",
    "    import spacy\n",
    "    import subprocess\n",
    "    import sys\n",
    "    subprocess.run([sys.executable,\n",
    "                    \"-m\",\n",
    "                    \"spacy\",\n",
    "                    \"init-model\",\n",
    "                    \"en\",\n",
    "                    model_name,\n",
    "                    \"--vectors-loc\",\n",
    "                    word_vectors\n",
    "                        ]\n",
    "                    )\n",
    "    print (f\"New spaCy model created with word vectors. File: {model_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "a2cfb11d",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/davidlin/ebay-umlc/predict.ipynb Cell 3\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu-22.04/home/davidlin/ebay-umlc/predict.ipynb#ch0000002vscode-remote?line=0'>1</a>\u001b[0m spacy\u001b[39m.\u001b[39mprefer_gpu()\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu-22.04/home/davidlin/ebay-umlc/predict.ipynb#ch0000002vscode-remote?line=1'>2</a>\u001b[0m \u001b[39m# load_word_vectors(\"./models/model-best\", \"dataset/word_vecs.txt\")\u001b[39;00m\n\u001b[0;32m----> <a href='vscode-notebook-cell://wsl%2Bubuntu-22.04/home/davidlin/ebay-umlc/predict.ipynb#ch0000002vscode-remote?line=2'>3</a>\u001b[0m nlp \u001b[39m=\u001b[39m spacy\u001b[39m.\u001b[39;49mload(\u001b[39m\"\u001b[39;49m\u001b[39m./models/model-best\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu-22.04/home/davidlin/ebay-umlc/predict.ipynb#ch0000002vscode-remote?line=3'>4</a>\u001b[0m dataframe \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mread_csv(\u001b[39m'\u001b[39m\u001b[39m./dataset/Train_Tagged_Titles.tsv\u001b[39m\u001b[39m'\u001b[39m, sep\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m\\t\u001b[39;00m\u001b[39m'\u001b[39m, on_bad_lines\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mskip\u001b[39m\u001b[39m'\u001b[39m, quoting\u001b[39m=\u001b[39mcsv\u001b[39m.\u001b[39mQUOTE_NONE, encoding\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mutf8\u001b[39m\u001b[39m'\u001b[39m)\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy/__init__.py:54\u001b[0m, in \u001b[0;36mload\u001b[0;34m(name, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mload\u001b[39m(\n\u001b[1;32m     31\u001b[0m     name: Union[\u001b[39mstr\u001b[39m, Path],\n\u001b[1;32m     32\u001b[0m     \u001b[39m*\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     37\u001b[0m     config: Union[Dict[\u001b[39mstr\u001b[39m, Any], Config] \u001b[39m=\u001b[39m util\u001b[39m.\u001b[39mSimpleFrozenDict(),\n\u001b[1;32m     38\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Language:\n\u001b[1;32m     39\u001b[0m     \u001b[39m\"\"\"Load a spaCy model from an installed package or a local path.\u001b[39;00m\n\u001b[1;32m     40\u001b[0m \n\u001b[1;32m     41\u001b[0m \u001b[39m    name (str): Package name or model path.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[39m    RETURNS (Language): The loaded nlp object.\u001b[39;00m\n\u001b[1;32m     53\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 54\u001b[0m     \u001b[39mreturn\u001b[39;00m util\u001b[39m.\u001b[39;49mload_model(\n\u001b[1;32m     55\u001b[0m         name,\n\u001b[1;32m     56\u001b[0m         vocab\u001b[39m=\u001b[39;49mvocab,\n\u001b[1;32m     57\u001b[0m         disable\u001b[39m=\u001b[39;49mdisable,\n\u001b[1;32m     58\u001b[0m         enable\u001b[39m=\u001b[39;49menable,\n\u001b[1;32m     59\u001b[0m         exclude\u001b[39m=\u001b[39;49mexclude,\n\u001b[1;32m     60\u001b[0m         config\u001b[39m=\u001b[39;49mconfig,\n\u001b[1;32m     61\u001b[0m     )\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy/util.py:431\u001b[0m, in \u001b[0;36mload_model\u001b[0;34m(name, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[1;32m    429\u001b[0m         \u001b[39mreturn\u001b[39;00m load_model_from_package(name, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[arg-type]\u001b[39;00m\n\u001b[1;32m    430\u001b[0m     \u001b[39mif\u001b[39;00m Path(name)\u001b[39m.\u001b[39mexists():  \u001b[39m# path to model data directory\u001b[39;00m\n\u001b[0;32m--> 431\u001b[0m         \u001b[39mreturn\u001b[39;00m load_model_from_path(Path(name), \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)  \u001b[39m# type: ignore[arg-type]\u001b[39;00m\n\u001b[1;32m    432\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mhasattr\u001b[39m(name, \u001b[39m\"\u001b[39m\u001b[39mexists\u001b[39m\u001b[39m\"\u001b[39m):  \u001b[39m# Path or Path-like to model data\u001b[39;00m\n\u001b[1;32m    433\u001b[0m     \u001b[39mreturn\u001b[39;00m load_model_from_path(name, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)  \u001b[39m# type: ignore[arg-type]\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy/util.py:511\u001b[0m, in \u001b[0;36mload_model_from_path\u001b[0;34m(model_path, meta, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[1;32m    502\u001b[0m config \u001b[39m=\u001b[39m load_config(config_path, overrides\u001b[39m=\u001b[39moverrides)\n\u001b[1;32m    503\u001b[0m nlp \u001b[39m=\u001b[39m load_model_from_config(\n\u001b[1;32m    504\u001b[0m     config,\n\u001b[1;32m    505\u001b[0m     vocab\u001b[39m=\u001b[39mvocab,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    509\u001b[0m     meta\u001b[39m=\u001b[39mmeta,\n\u001b[1;32m    510\u001b[0m )\n\u001b[0;32m--> 511\u001b[0m \u001b[39mreturn\u001b[39;00m nlp\u001b[39m.\u001b[39;49mfrom_disk(model_path, exclude\u001b[39m=\u001b[39;49mexclude, overrides\u001b[39m=\u001b[39;49moverrides)\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy/language.py:2115\u001b[0m, in \u001b[0;36mLanguage.from_disk\u001b[0;34m(self, path, exclude, overrides)\u001b[0m\n\u001b[1;32m   2112\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (path \u001b[39m/\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mvocab\u001b[39m\u001b[39m\"\u001b[39m)\u001b[39m.\u001b[39mexists() \u001b[39mand\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mvocab\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m exclude:  \u001b[39m# type: ignore[operator]\u001b[39;00m\n\u001b[1;32m   2113\u001b[0m     \u001b[39m# Convert to list here in case exclude is (default) tuple\u001b[39;00m\n\u001b[1;32m   2114\u001b[0m     exclude \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(exclude) \u001b[39m+\u001b[39m [\u001b[39m\"\u001b[39m\u001b[39mvocab\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[0;32m-> 2115\u001b[0m util\u001b[39m.\u001b[39;49mfrom_disk(path, deserializers, exclude)  \u001b[39m# type: ignore[arg-type]\u001b[39;00m\n\u001b[1;32m   2116\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_path \u001b[39m=\u001b[39m path  \u001b[39m# type: ignore[assignment]\u001b[39;00m\n\u001b[1;32m   2117\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_link_components()\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy/util.py:1340\u001b[0m, in \u001b[0;36mfrom_disk\u001b[0;34m(path, readers, exclude)\u001b[0m\n\u001b[1;32m   1337\u001b[0m \u001b[39mfor\u001b[39;00m key, reader \u001b[39min\u001b[39;00m readers\u001b[39m.\u001b[39mitems():\n\u001b[1;32m   1338\u001b[0m     \u001b[39m# Split to support file names like meta.json\u001b[39;00m\n\u001b[1;32m   1339\u001b[0m     \u001b[39mif\u001b[39;00m key\u001b[39m.\u001b[39msplit(\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m)[\u001b[39m0\u001b[39m] \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m exclude:\n\u001b[0;32m-> 1340\u001b[0m         reader(path \u001b[39m/\u001b[39;49m key)\n\u001b[1;32m   1341\u001b[0m \u001b[39mreturn\u001b[39;00m path\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy/language.py:2109\u001b[0m, in \u001b[0;36mLanguage.from_disk.<locals>.<lambda>\u001b[0;34m(p, proc)\u001b[0m\n\u001b[1;32m   2107\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mhasattr\u001b[39m(proc, \u001b[39m\"\u001b[39m\u001b[39mfrom_disk\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[1;32m   2108\u001b[0m         \u001b[39mcontinue\u001b[39;00m\n\u001b[0;32m-> 2109\u001b[0m     deserializers[name] \u001b[39m=\u001b[39m \u001b[39mlambda\u001b[39;00m p, proc\u001b[39m=\u001b[39mproc: proc\u001b[39m.\u001b[39;49mfrom_disk(  \u001b[39m# type: ignore[misc]\u001b[39;49;00m\n\u001b[1;32m   2110\u001b[0m         p, exclude\u001b[39m=\u001b[39;49m[\u001b[39m\"\u001b[39;49m\u001b[39mvocab\u001b[39;49m\u001b[39m\"\u001b[39;49m]\n\u001b[1;32m   2111\u001b[0m     )\n\u001b[1;32m   2112\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (path \u001b[39m/\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mvocab\u001b[39m\u001b[39m\"\u001b[39m)\u001b[39m.\u001b[39mexists() \u001b[39mand\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mvocab\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m exclude:  \u001b[39m# type: ignore[operator]\u001b[39;00m\n\u001b[1;32m   2113\u001b[0m     \u001b[39m# Convert to list here in case exclude is (default) tuple\u001b[39;00m\n\u001b[1;32m   2114\u001b[0m     exclude \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(exclude) \u001b[39m+\u001b[39m [\u001b[39m\"\u001b[39m\u001b[39mvocab\u001b[39m\u001b[39m\"\u001b[39m]\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy_transformers/pipeline_component.py:420\u001b[0m, in \u001b[0;36mTransformer.from_disk\u001b[0;34m(self, path, exclude)\u001b[0m\n\u001b[1;32m    413\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mattrs[\u001b[39m\"\u001b[39m\u001b[39mset_transformer\u001b[39m\u001b[39m\"\u001b[39m](\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel, hf_model)\n\u001b[1;32m    415\u001b[0m deserialize \u001b[39m=\u001b[39m {\n\u001b[1;32m    416\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mvocab\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvocab\u001b[39m.\u001b[39mfrom_disk,\n\u001b[1;32m    417\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mcfg\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m p: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcfg\u001b[39m.\u001b[39mupdate(deserialize_config(p)),\n\u001b[1;32m    418\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mmodel\u001b[39m\u001b[39m\"\u001b[39m: load_model,\n\u001b[1;32m    419\u001b[0m }\n\u001b[0;32m--> 420\u001b[0m util\u001b[39m.\u001b[39;49mfrom_disk(path, deserialize, exclude)\n\u001b[1;32m    421\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy/util.py:1340\u001b[0m, in \u001b[0;36mfrom_disk\u001b[0;34m(path, readers, exclude)\u001b[0m\n\u001b[1;32m   1337\u001b[0m \u001b[39mfor\u001b[39;00m key, reader \u001b[39min\u001b[39;00m readers\u001b[39m.\u001b[39mitems():\n\u001b[1;32m   1338\u001b[0m     \u001b[39m# Split to support file names like meta.json\u001b[39;00m\n\u001b[1;32m   1339\u001b[0m     \u001b[39mif\u001b[39;00m key\u001b[39m.\u001b[39msplit(\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m)[\u001b[39m0\u001b[39m] \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m exclude:\n\u001b[0;32m-> 1340\u001b[0m         reader(path \u001b[39m/\u001b[39;49m key)\n\u001b[1;32m   1341\u001b[0m \u001b[39mreturn\u001b[39;00m path\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy_transformers/pipeline_component.py:394\u001b[0m, in \u001b[0;36mTransformer.from_disk.<locals>.load_model\u001b[0;34m(p)\u001b[0m\n\u001b[1;32m    392\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    393\u001b[0m     \u001b[39mwith\u001b[39;00m \u001b[39mopen\u001b[39m(p, \u001b[39m\"\u001b[39m\u001b[39mrb\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39mas\u001b[39;00m mfile:\n\u001b[0;32m--> 394\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmodel\u001b[39m.\u001b[39;49mfrom_bytes(mfile\u001b[39m.\u001b[39;49mread())\n\u001b[1;32m    395\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mAttributeError\u001b[39;00m:\n\u001b[1;32m    396\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(Errors\u001b[39m.\u001b[39mE149) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39m\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/thinc/model.py:619\u001b[0m, in \u001b[0;36mModel.from_bytes\u001b[0;34m(self, bytes_data)\u001b[0m\n\u001b[1;32m    617\u001b[0m msg \u001b[39m=\u001b[39m srsly\u001b[39m.\u001b[39mmsgpack_loads(bytes_data)\n\u001b[1;32m    618\u001b[0m msg \u001b[39m=\u001b[39m convert_recursive(is_xp_array, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mops\u001b[39m.\u001b[39masarray, msg)\n\u001b[0;32m--> 619\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mfrom_dict(msg)\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/thinc/model.py:657\u001b[0m, in \u001b[0;36mModel.from_dict\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m    655\u001b[0m         node\u001b[39m.\u001b[39mset_param(param_name, value)\n\u001b[1;32m    656\u001b[0m     \u001b[39mfor\u001b[39;00m i, shim_bytes \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(msg[\u001b[39m\"\u001b[39m\u001b[39mshims\u001b[39m\u001b[39m\"\u001b[39m][i]):\n\u001b[0;32m--> 657\u001b[0m         node\u001b[39m.\u001b[39;49mshims[i]\u001b[39m.\u001b[39;49mfrom_bytes(shim_bytes)\n\u001b[1;32m    658\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/spacy_transformers/layers/hf_shim.py:123\u001b[0m, in \u001b[0;36mHFShim.from_bytes\u001b[0;34m(self, bytes_data)\u001b[0m\n\u001b[1;32m    121\u001b[0m         device_id \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mcuda\u001b[39m.\u001b[39mcurrent_device()\n\u001b[1;32m    122\u001b[0m         map_location \u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mcuda:\u001b[39m\u001b[39m{\u001b[39;00mdevice_id\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[0;32m--> 123\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_model\u001b[39m.\u001b[39mload_state_dict(torch\u001b[39m.\u001b[39;49mload(filelike, map_location\u001b[39m=\u001b[39;49mmap_location))\n\u001b[1;32m    124\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_model\u001b[39m.\u001b[39mto(map_location)\n\u001b[1;32m    125\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/torch/serialization.py:712\u001b[0m, in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m    710\u001b[0m             opened_file\u001b[39m.\u001b[39mseek(orig_position)\n\u001b[1;32m    711\u001b[0m             \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39mjit\u001b[39m.\u001b[39mload(opened_file)\n\u001b[0;32m--> 712\u001b[0m         \u001b[39mreturn\u001b[39;00m _load(opened_zipfile, map_location, pickle_module, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mpickle_load_args)\n\u001b[1;32m    713\u001b[0m \u001b[39mreturn\u001b[39;00m _legacy_load(opened_file, map_location, pickle_module, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mpickle_load_args)\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/torch/serialization.py:1049\u001b[0m, in \u001b[0;36m_load\u001b[0;34m(zip_file, map_location, pickle_module, pickle_file, **pickle_load_args)\u001b[0m\n\u001b[1;32m   1047\u001b[0m unpickler \u001b[39m=\u001b[39m UnpicklerWrapper(data_file, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mpickle_load_args)\n\u001b[1;32m   1048\u001b[0m unpickler\u001b[39m.\u001b[39mpersistent_load \u001b[39m=\u001b[39m persistent_load\n\u001b[0;32m-> 1049\u001b[0m result \u001b[39m=\u001b[39m unpickler\u001b[39m.\u001b[39;49mload()\n\u001b[1;32m   1051\u001b[0m torch\u001b[39m.\u001b[39m_utils\u001b[39m.\u001b[39m_validate_loaded_sparse_tensors()\n\u001b[1;32m   1053\u001b[0m \u001b[39mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/torch/serialization.py:1019\u001b[0m, in \u001b[0;36m_load.<locals>.persistent_load\u001b[0;34m(saved_id)\u001b[0m\n\u001b[1;32m   1017\u001b[0m \u001b[39mif\u001b[39;00m key \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m loaded_storages:\n\u001b[1;32m   1018\u001b[0m     nbytes \u001b[39m=\u001b[39m numel \u001b[39m*\u001b[39m torch\u001b[39m.\u001b[39m_utils\u001b[39m.\u001b[39m_element_size(dtype)\n\u001b[0;32m-> 1019\u001b[0m     load_tensor(dtype, nbytes, key, _maybe_decode_ascii(location))\n\u001b[1;32m   1021\u001b[0m \u001b[39mreturn\u001b[39;00m loaded_storages[key]\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/torch/serialization.py:1001\u001b[0m, in \u001b[0;36m_load.<locals>.load_tensor\u001b[0;34m(dtype, numel, key, location)\u001b[0m\n\u001b[1;32m    997\u001b[0m storage \u001b[39m=\u001b[39m zip_file\u001b[39m.\u001b[39mget_storage_from_record(name, numel, torch\u001b[39m.\u001b[39m_UntypedStorage)\u001b[39m.\u001b[39mstorage()\u001b[39m.\u001b[39m_untyped()\n\u001b[1;32m    998\u001b[0m \u001b[39m# TODO: Once we decide to break serialization FC, we can\u001b[39;00m\n\u001b[1;32m    999\u001b[0m \u001b[39m# stop wrapping with _TypedStorage\u001b[39;00m\n\u001b[1;32m   1000\u001b[0m loaded_storages[key] \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mstorage\u001b[39m.\u001b[39m_TypedStorage(\n\u001b[0;32m-> 1001\u001b[0m     wrap_storage\u001b[39m=\u001b[39mrestore_location(storage, location),\n\u001b[1;32m   1002\u001b[0m     dtype\u001b[39m=\u001b[39mdtype)\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/torch/serialization.py:970\u001b[0m, in \u001b[0;36m_get_restore_location.<locals>.restore_location\u001b[0;34m(storage, location)\u001b[0m\n\u001b[1;32m    969\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrestore_location\u001b[39m(storage, location):\n\u001b[0;32m--> 970\u001b[0m     \u001b[39mreturn\u001b[39;00m default_restore_location(storage, map_location)\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/torch/serialization.py:175\u001b[0m, in \u001b[0;36mdefault_restore_location\u001b[0;34m(storage, location)\u001b[0m\n\u001b[1;32m    173\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdefault_restore_location\u001b[39m(storage, location):\n\u001b[1;32m    174\u001b[0m     \u001b[39mfor\u001b[39;00m _, _, fn \u001b[39min\u001b[39;00m _package_registry:\n\u001b[0;32m--> 175\u001b[0m         result \u001b[39m=\u001b[39m fn(storage, location)\n\u001b[1;32m    176\u001b[0m         \u001b[39mif\u001b[39;00m result \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    177\u001b[0m             \u001b[39mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/torch/serialization.py:157\u001b[0m, in \u001b[0;36m_cuda_deserialize\u001b[0;34m(obj, location)\u001b[0m\n\u001b[1;32m    155\u001b[0m         \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39m_UntypedStorage(obj\u001b[39m.\u001b[39mnbytes(), device\u001b[39m=\u001b[39mtorch\u001b[39m.\u001b[39mdevice(location))\n\u001b[1;32m    156\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 157\u001b[0m     \u001b[39mreturn\u001b[39;00m obj\u001b[39m.\u001b[39;49mcuda(device)\n",
      "File \u001b[0;32m~/anaconda3/envs/poopy/lib/python3.10/site-packages/torch/_utils.py:78\u001b[0m, in \u001b[0;36m_cuda\u001b[0;34m(self, device, non_blocking, **kwargs)\u001b[0m\n\u001b[1;32m     76\u001b[0m     \u001b[39mreturn\u001b[39;00m new_type(indices, values, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msize())\n\u001b[1;32m     77\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 78\u001b[0m     \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39;49m_UntypedStorage(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msize(), device\u001b[39m=\u001b[39;49mtorch\u001b[39m.\u001b[39;49mdevice(\u001b[39m'\u001b[39;49m\u001b[39mcuda\u001b[39;49m\u001b[39m'\u001b[39;49m))\u001b[39m.\u001b[39;49mcopy_(\u001b[39mself\u001b[39;49m, non_blocking)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "spacy.prefer_gpu()\n",
    "# load_word_vectors(\"./models/model-best\", \"dataset/word_vecs.txt\")\n",
    "nlp = spacy.load(\"./models/model-best\")\n",
    "dataframe = pd.read_csv('./dataset/Train_Tagged_Titles.tsv', sep='\\t', on_bad_lines='skip', quoting=csv.QUOTE_NONE, encoding='utf8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bf36df6",
   "metadata": {},
   "outputs": [],
   "source": [
    "listing_titles = [(dataframe[\"Title\"][i], i) for i in range(0, len(dataframe[\"Title\"]))][5000:30000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef42f47e",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_titles = [(dataframe[\"Title\"][i], i) for i in range(0, len(dataframe[\"Title\"]))][-100:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21e0f829",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 100/25000 [00:00<02:01, 205.14it/s]\n"
     ]
    }
   ],
   "source": [
    "result = []\n",
    "\n",
    "with tqdm(total = 25000) as pbar:\n",
    "    buffer = \"\"\n",
    "    for doc, i in nlp.pipe(input_titles, as_tuples=True, batch_size=10000):\n",
    "        for ent in doc.ents:\n",
    "                \n",
    "                #have to extract text from listing_titles[i]\n",
    "                #find the similar text then delete it\n",
    "                #have to add punctuation back if possible\n",
    "                \n",
    "                result.append(str(i + 1) + '\\t' + ent.label_ + '\\t' + ent.text)\n",
    "        pbar.update(1)\n",
    "        \n",
    "#output data\n",
    "os.makedirs(\"./output\", exist_ok=True)\n",
    "output = open(\"output/output.tsv\", \"w\", encoding=\"utf-8\")  # write mode\n",
    "output.write('\\n'.join(result))\n",
    "output.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('poopy')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "vscode": {
   "interpreter": {
    "hash": "f52ab0dcc20aaef27cce08a77f09e222b8b3949b76637bac784c92d7f6a2f51c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}