Teacher Forcing Datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import scipy\n",
    "import scipy.stats\n",
    "import sklearn\n",
    "import sklearn.feature_extraction, sklearn.naive_bayes, sklearn.metrics, sklearn.externals\n",
    "from collections import defaultdict, Counter\n",
    "\n",
    "from dataset import SpamDataset\n",
    "from discriminator import MultinomialNBDiscriminator\n",
    "\n",
    "# The number of terms, including special tokens, in the final vocabulary.\n",
    "TRAINING_VOCAB_SIZE = {\n",
    "    100: 4480,\n",
    "    30: 4628\n",
    "} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now on truncation_len: 30\n",
      "Mean_accuracy: 0.021549\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Demonstration models.\n",
    "\n",
    "truncation_len=30\n",
    "print \"Now on truncation_len: \" + str(truncation_len)\n",
    "spam_dataset = SpamDataset(truncation_len=truncation_len, label_names=['spam'], encoded_files=['encoded_spam.txt'])\n",
    "\n",
    "checkpoints_dir = '/cvgl2/u/catwong/cs332_final_project/checkpoints'\n",
    "checkpoint_files = [\"_multinomial_nb_model.pkl\", \"_multinomial_nb_tf_transformer.pkl\"]\n",
    "checkpoint = [os.path.join(checkpoints_dir, str(truncation_len) + filename) for filename in checkpoint_files]\n",
    "new_discriminator = MultinomialNBDiscriminator(checkpoint=checkpoint, truncation_len=truncation_len)\n",
    "examples, labels, probs = new_discriminator.evaluate(spam_dataset, 'val')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0.00000000e+00   5.61000000e+02   2.00000000e+00   1.52900000e+03\n",
      "    9.65000000e+02   2.00000000e+00   7.40000000e+02   2.00000000e+00\n",
      "    1.41200000e+03   1.88700000e+03   3.19000000e+02   2.00000000e+00\n",
      "    3.19000000e+02   1.52900000e+03   9.65000000e+02   1.41200000e+03\n",
      "    2.00000000e+00   7.40000000e+02   2.09300000e+03   2.27200000e+03\n",
      "    4.11000000e+02   2.00000000e+00   2.00000000e+00   2.00000000e+00\n",
      "    2.00000000e+00   2.00000000e+00   2.00000000e+00   2.00000000e+00\n",
      "    2.00000000e+00   2.00000000e+00   2.00000000e+00   1.00000000e+00\n",
      "    5.34501102e-07   9.99999465e-01]\n",
      " [  0.00000000e+00   5.61000000e+02   1.72500000e+03   2.00000000e+00\n",
      "    4.31200000e+03   2.00000000e+00   1.39800000e+03   7.75000000e+02\n",
      "    3.49000000e+02   4.32700000e+03   7.75000000e+02   3.47800000e+03\n",
      "    2.34200000e+03   7.75000000e+02   4.32700000e+03   1.28300000e+03\n",
      "    3.47800000e+03   1.68800000e+03   1.88700000e+03   4.32700000e+03\n",
      "    2.10800000e+03   3.20000000e+03   3.26000000e+03   4.32700000e+03\n",
      "    1.23400000e+03   3.49000000e+02   1.39800000e+03   5.50000000e+02\n",
      "    2.62100000e+03   1.88700000e+03   3.26000000e+03   1.00000000e+00\n",
      "    4.99197903e-13   1.00000000e+00]\n",
      " [  0.00000000e+00   5.61000000e+02   6.02000000e+02   1.52500000e+03\n",
      "    4.00000000e+03   2.03800000e+03   3.19000000e+02   1.80500000e+03\n",
      "    4.47300000e+03   2.92900000e+03   3.66400000e+03   1.12000000e+02\n",
      "    2.58400000e+03   4.47300000e+03   2.00000000e+00   7.40000000e+02\n",
      "    7.40000000e+02   7.40000000e+02   7.40000000e+02   2.00000000e+00\n",
      "    3.87800000e+03   2.10800000e+03   7.90000000e+02   3.62600000e+03\n",
      "    4.44000000e+03   1.44000000e+03   9.36000000e+02   3.07100000e+03\n",
      "    2.23800000e+03   1.97000000e+02   8.53000000e+02   1.00000000e+00\n",
      "    8.32901504e-10   9.99999999e-01]\n",
      " [  0.00000000e+00   5.61000000e+02   3.79300000e+03   2.00000000e+00\n",
      "    3.83000000e+02   2.00000000e+00   2.00000000e+00   2.00000000e+00\n",
      "    2.00000000e+00   2.00000000e+00   2.00000000e+00   2.00000000e+00\n",
      "    2.00000000e+00   2.00000000e+00   2.00000000e+00   2.00000000e+00\n",
      "    2.00000000e+00   9.37000000e+02   2.00000000e+00   2.00000000e+00\n",
      "    2.00000000e+00   2.00000000e+00   3.01900000e+03   2.98000000e+03\n",
      "    1.97000000e+02   2.17300000e+03   3.70700000e+03   2.41400000e+03\n",
      "    2.00000000e+00   2.00000000e+00   2.00000000e+00   1.00000000e+00\n",
      "    1.38629324e-06   9.99998614e-01]\n",
      " [  0.00000000e+00   5.61000000e+02   2.60700000e+03   3.38600000e+03\n",
      "    4.32700000e+03   3.87800000e+03   5.50000000e+02   2.00000000e+00\n",
      "    7.40000000e+02   7.98000000e+02   2.00000000e+00   1.52500000e+03\n",
      "    2.27200000e+03   2.35200000e+03   2.00000000e+00   3.00400000e+03\n",
      "    1.12000000e+02   2.00000000e+00   4.15400000e+03   3.00000000e+00\n",
      "    3.00000000e+00   3.00000000e+00   3.00000000e+00   3.00000000e+00\n",
      "    3.00000000e+00   3.00000000e+00   3.00000000e+00   3.00000000e+00\n",
      "    3.00000000e+00   3.00000000e+00   3.00000000e+00   1.00000000e+00\n",
      "    4.40094445e-02   9.55990555e-01]]\n",
      "[[  0.00000000e+00   5.61000000e+02   2.61200000e+03   2.00000000e+00\n",
      "    2.17300000e+03   3.63100000e+03   1.45300000e+03   2.17300000e+03\n",
      "    8.70000000e+02   4.38500000e+03   3.66400000e+03   3.38600000e+03\n",
      "    2.14000000e+02   3.89500000e+03   2.00000000e+00   2.01100000e+03\n",
      "    3.19000000e+02   4.03900000e+03   9.65000000e+02   2.96000000e+02\n",
      "    3.90000000e+02   3.38600000e+03   2.61200000e+03   2.00000000e+00\n",
      "    2.17300000e+03   3.63100000e+03   1.45300000e+03   2.17300000e+03\n",
      "    8.70000000e+02   4.38500000e+03   4.32700000e+03   1.00000000e+00\n",
      "    9.99999983e-01   1.73220944e-08]\n",
      " [  0.00000000e+00   5.61000000e+02   3.38600000e+03   3.38600000e+03\n",
      "    3.38600000e+03   3.38600000e+03   3.38600000e+03   3.38600000e+03\n",
      "    3.38600000e+03   3.38600000e+03   3.38600000e+03   3.38600000e+03\n",
      "    3.38600000e+03   4.39500000e+03   2.46300000e+03   5.17000000e+02\n",
      "    2.62100000e+03   5.17000000e+02   2.00000000e+00   3.38600000e+03\n",
      "    3.38600000e+03   3.38600000e+03   3.38600000e+03   3.38600000e+03\n",
      "    3.38600000e+03   3.38600000e+03   3.38600000e+03   3.38600000e+03\n",
      "    3.38600000e+03   3.38600000e+03   2.00000000e+00   1.00000000e+00\n",
      "    9.99999776e-01   2.24354853e-07]\n",
      " [  0.00000000e+00   5.61000000e+02   4.37700000e+03   3.37400000e+03\n",
      "    3.72100000e+03   2.00000000e+00   2.00000000e+00   1.50300000e+03\n",
      "    4.55300000e+03   5.17000000e+02   5.17000000e+02   5.17000000e+02\n",
      "    5.17000000e+02   5.17000000e+02   5.17000000e+02   5.17000000e+02\n",
      "    5.17000000e+02   5.17000000e+02   5.17000000e+02   5.17000000e+02\n",
      "    5.17000000e+02   5.17000000e+02   5.17000000e+02   5.17000000e+02\n",
      "    5.17000000e+02   5.17000000e+02   5.17000000e+02   5.17000000e+02\n",
      "    5.17000000e+02   5.17000000e+02   5.17000000e+02   1.00000000e+00\n",
      "    9.99999663e-01   3.36952039e-07]\n",
      " [  0.00000000e+00   5.61000000e+02   3.90000000e+02   3.38600000e+03\n",
      "    2.20800000e+03   2.00000000e+00   2.62000000e+03   2.44900000e+03\n",
      "    3.19000000e+02   1.67600000e+03   6.00000000e+02   3.48900000e+03\n",
      "    6.55000000e+02   3.38600000e+03   1.68300000e+03   3.38600000e+03\n",
      "    3.63100000e+03   5.17000000e+02   4.40100000e+03   2.15700000e+03\n",
      "    8.02000000e+02   3.38600000e+03   3.01100000e+03   2.29200000e+03\n",
      "    2.20800000e+03   1.08900000e+03   3.72600000e+03   7.88000000e+02\n",
      "    4.27100000e+03   2.22900000e+03   7.40000000e+02   1.00000000e+00\n",
      "    9.99024460e-01   9.75540136e-04]\n",
      " [  0.00000000e+00   5.61000000e+02   3.90000000e+02   3.38600000e+03\n",
      "    3.86600000e+03   4.63000000e+02   3.68600000e+03   2.47200000e+03\n",
      "    1.64900000e+03   2.38100000e+03   2.08700000e+03   3.19000000e+02\n",
      "    1.67900000e+03   1.52700000e+03   3.48900000e+03   6.62000000e+02\n",
      "    3.38600000e+03   1.68400000e+03   3.38600000e+03   1.67600000e+03\n",
      "    5.17000000e+02   4.40100000e+03   1.59000000e+03   5.17000000e+02\n",
      "    2.45400000e+03   3.38600000e+03   1.41200000e+03   7.40000000e+02\n",
      "    1.19300000e+03   2.37000000e+03   5.17000000e+02   1.00000000e+00\n",
      "    9.97760497e-01   2.23950319e-03]]\n"
     ]
    }
   ],
   "source": [
    "# Join and sort by spam probability\n",
    "examples_probs = np.hstack((examples, probs))\n",
    "examples_sorted = examples_probs[examples_probs[:,-1].argsort()]\n",
    "print examples_probs[:5]\n",
    "print examples_sorted[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"<SOS> subject: <UNK> , <UNK> is poor men ' s wealth . when i was doing ' this boy ' s life ' i wanted to be as old as robert <EOS>\""
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spam_dataset.vocab_encoder.decode_numpy(examples_sorted[171][:-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/cvgl2/u/catwong/cs332_final_project/data/train/30_encoded_spam_low_conf.txt\n",
      "/cvgl2/u/catwong/cs332_final_project/data/val/30_encoded_spam_low_conf.txt\n",
      "/cvgl2/u/catwong/cs332_final_project/data/test/30_encoded_spam_low_conf.txt\n"
     ]
    }
   ],
   "source": [
    "# Write out the examples to disk.\n",
    "base_data_dir = \"/cvgl2/u/catwong/cs332_final_project/data/\"\n",
    "splits = ['train', 'val', 'test']\n",
    "classes = ['encoded_spam_low_conf.txt']\n",
    "split = splits[1]\n",
    "class_name = classes[0]\n",
    "\n",
    "low_conf = int(examples_sorted.shape[0] * .1)\n",
    "low_conf_examples = examples_sorted[:low_conf][:,:-2].astype(int)\n",
    "for split in splits:\n",
    "    filename = os.path.join(base_data_dir, split, str(truncation_len) + \"_\" + class_name)\n",
    "    print filename\n",
    "    np.savetxt(filename, low_conf_examples, fmt=\"%d\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "low_conf_dataset = SpamDataset(truncation_len=truncation_len, encoded_files=['encoded_spam_low_conf.txt'])\n",
    "examples, _ = low_conf_dataset.examples(split=\"val\", shuffled=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}