cluster_20.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clustering the 20NewsGroup dataset\n",
    "\n",
    "### Author: Stephen W. Thomas, Queen's University\n",
    "\n",
    "### Steps\n",
    "\n",
    "- Load data (from the scikit-learn package)\n",
    "- Select subset of data\n",
    "- Preprocess data\n",
    "- Transform data into a term document matrix (TDM)\n",
    "- Apply TF-IDF to TDM\n",
    "- Run K-Means clustering on TDM\n",
    "- Measure effectiveness\n",
    "- Output results and examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/stepthom/anaconda/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import re\n",
    "import sys\n",
    "\n",
    "import nltk.corpus\n",
    "import nltk.stem\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from scipy.spatial.distance import cdist, pdist\n",
    "import textwrap\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of docs: 3582:\n",
      "Number of categories: 4\n",
      "Category 0 (comp.graphics): 973 docs\n",
      "Category 1 (rec.sport.baseball): 994 docs\n",
      "Category 2 (sci.space): 987 docs\n",
      "Category 3 (talk.religion.misc): 628 docs\n"
     ]
    }
   ],
   "source": [
    "# For now, we'll just use these categories\n",
    "categories = ['rec.sport.baseball', 'talk.religion.misc', 'comp.graphics','sci.space']\n",
    "data = fetch_20newsgroups(subset='all', categories=categories, shuffle=False)\n",
    "\n",
    "# Print some metrics about the data\n",
    "print(\"Number of docs: %d:\" % len(data.data))\n",
    "print(\"Number of categories: %d\" % len(data.target_names))\n",
    "i = 0\n",
    "for cat in range(len(data.target_names)):\n",
    "    cat_name = data.target_names[cat]\n",
    "    num_docs = len([d for d in data.target if d == i])\n",
    "    print(\"Category %d (%s): %d docs\" % (i, cat_name, num_docs))\n",
    "    i += 1\n",
    "\n",
    "num_categories = np.unique(categories).shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Define a new member, .pre, to hold the preprocessed version.\n",
    "data.pre = []\n",
    "\n",
    "# A list of custom stopwords to remove\n",
    "stopwords = set(nltk.corpus.stopwords.words('english'))\n",
    "stopwords.update(\n",
    "    ['would', 'subject', 're', 'don', 'jan',\n",
    "     'feb', 'mar', 'apr', 'may', 'june',\n",
    "     'july', 'aug', 'sep', 'oct', 'nov', 'dec'])\n",
    "\n",
    "# Regex pattern for email addresses\n",
    "email_pattern = re.compile(r'[^@]+@[^@]+\\.[^@]+', re.IGNORECASE & re.UNICODE)\n",
    "\n",
    "# Regex pattern for things found at the top \"header\" part of the message\n",
    "header_pattern = re.compile(\n",
    "    r'^summary:|^x-newsreader:.*|^date:.*'\n",
    "    r'|^disclaimer:.*|^distribution:.*|^organization:.*'\n",
    "    r'|^nntp-posting-host:.*|^keywords:.*|^to:.*'\n",
    "    r'|^in-reply-to:.*|^x-news-reader:.*|^lines:.*',\n",
    "    re.IGNORECASE & re.UNICODE)\n",
    "\n",
    "# Regex pattern for general special characters\n",
    "#special_char_pattern = re.compile(r'[\\|=-\\[\\]\\'\\\":;,\\.\\<\\>\\\\\\/\\?_\\(\\)!$%^&*,]', re.IGNORECASE & re.UNICODE)\n",
    "special_char_pattern = re.compile(r\"[^\\w']|_\")\n",
    "\n",
    "# Regex pattern for whole-word numbers\n",
    "number_pattern = re.compile(r'\\b\\d+\\b')\n",
    "\n",
    "# Regex pattern to handle strings like the following:\n",
    "# In article <me@me.com> me@me.com (Me) writes:\n",
    "writes_pattern = re.compile(r'^.*writes:$')\n",
    "\n",
    "# Loop through data do the preprocessing\n",
    "for j in range(0, len(data.data)):\n",
    "    lines = data.data[j].lower().split(\"\\n\")\n",
    "    for i in range(0, len(lines)):\n",
    "\n",
    "        # Use the regexes above to remove bad things\n",
    "        lines[i] = header_pattern.sub(' ', lines[i])\n",
    "        lines[i] = email_pattern.sub(' ', lines[i])\n",
    "        lines[i] = number_pattern.sub(' ', lines[i])\n",
    "        lines[i] = writes_pattern.sub(' ', lines[i])\n",
    "        lines[i] = special_char_pattern.sub(' ', lines[i])\n",
    "\n",
    "        # Remove short words\n",
    "        lines[i] = ' '.join([w for w in lines[i].split() if len(w) > 2])\n",
    "\n",
    "        # Remove stopwords\n",
    "        lines[i] = ' '.join([w for w in lines[i].split() if w not in stopwords])\n",
    "\n",
    "        # Stem the words\n",
    "        lines[i] = ' '.join([nltk.stem.snowball.SnowballStemmer(\"english\").stem(w) for w in lines[i].split()])\n",
    "\n",
    "        # Remove extra spaces, just for beauty\n",
    "        lines[i] = re.sub(r'\\s\\s+', \" \", lines[i])\n",
    "\n",
    "    pre = \" \".join(lines)\n",
    "    data.pre.append(pre)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspect some of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " cub behind marlin articl agat 1pt592 f9a        morgan guzman era run higher last year cub idiot pitch harkey much hibbard castillo won't good think he stud pitcher  season far morgan guzman help lead cub top era even better rotat atlanta cub era brave know earli season cub fan learn enjoy short triumph still \n",
      "\n",
      " time best player origin news nimast         articl 1993apr13 write  i'v recent work project determin greatest player respect postion  career  johnni hodapp greatest 2nd baseman cleveland indian histori hit consist great great second baseman  header said career hodapp manag nine year major  consist make three year row preced part time year plus last year boston hodapp qualifi bat titl five time  injur retir right around 28th birthday  anyway hodapp put flashi number year everybodi put flashi number realli good year bat year mar lack power inabl draw walk base ball describ fear hitter  hand part long line famous cleveland wambsganss rigg stephenson etc  roger  jon johnni hodapp      \n",
      "482: >a few years ago a friend and i took some 256 grey-level photos from\n",
      "1289: a few years ago a friend and i took some 256 grey-level photos from\n",
      "\n",
      " voic input vendor inform need      hello look add voic input capabl user interfac develop hp730 unix workstat great appreci inform anyon care offer voic input system easili access unix environ  name adress applic vendor well experi specif system help  pleas respond via email post summari suffici interest   thank ken   found sever impress system ibm pc like avoid hassl purchas maintain separ possibl    univers virginia neurosurg visual laboratori  \n",
      "\n",
      " gray level screen      year ago friend took grey level photo bit mac plus screen use method display level synchron 60hz display took second  didn't creat grey level imag display time slice  requir total exposur time precis time drastic reduc imag prepar time        kenneth sloan comput inform scienc  115a campbel hall uab station fax birmingham \n"
     ]
    }
   ],
   "source": [
    "print(data.pre[0])\n",
    "print(\"\")\n",
    "print(data.pre[1])\n",
    "\n",
    "# Which doc is our example in the slides?\n",
    "for j in range(0, len(data.data)):\n",
    "    lines = data.data[j].lower().split(\"\\n\")\n",
    "    for i in range(0, len(lines)):\n",
    "        if \"a few years ago a friend and i took\" in lines[i]:\n",
    "            print(\"{}: {}\".format(j, lines[i]))\n",
    "            \n",
    "print(\"\")\n",
    "print(data.pre[1129])\n",
    "            \n",
    "print(\"\")\n",
    "print(data.pre[482])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total terms: 476206\n",
      "Unique terms: 23675\n",
      "Term frequencies:\n",
      "  2689, one\n",
      "  2417, use\n",
      "  2225, imag\n",
      "  2039, space\n",
      "  2030, like\n",
      "  1841, get\n",
      "  1795, year\n",
      "  1736, don't\n",
      "  1694, know\n",
      "  1661, time\n",
      "  1608, think\n",
      "  1566, also\n",
      "  1404, say\n",
      "  1316, file\n",
      "  1315, see\n",
      "  1314, make\n",
      "  1303, good\n",
      "  1288, peopl\n",
      "  1234, look\n",
      "  1207, game\n",
      "  1193, graphic\n",
      "  1188, program\n",
      "  1185, system\n",
      "  1179, it\n",
      "  1160, could\n",
      "  1160, first\n",
      "  1144, god\n",
      "  1124, well\n",
      "  1063, point\n",
      "  1051, even\n",
      "  1046, much\n",
      "  1031, run\n",
      "  1026, want\n",
      "  1015, way\n",
      "  1008, work\n",
      "  1000, i'm\n",
      "   993, need\n",
      "   986, thing\n",
      "   970, two\n",
      "   949, mani\n",
      "   937, new\n",
      "   896, data\n",
      "   861, bit\n",
      "   850, post\n",
      "   837, base\n",
      "   829, last\n",
      "   828, jpeg\n",
      "   820, team\n",
      "   819, orbit\n",
      "   817, call\n",
      "   814, better\n",
      "   811, softwar\n",
      "   810, read\n",
      "   801, come\n",
      "   800, christian\n",
      "   800, right\n",
      "   781, format\n",
      "   778, find\n",
      "   776, day\n",
      "   768, take\n",
      "   764, avail\n",
      "   751, anyon\n",
      "   748, mean\n",
      "   748, edu\n",
      "   741, tri\n",
      "   714, includ\n",
      "   712, give\n",
      "   711, back\n",
      "   709, hit\n",
      "   707, help\n",
      "   704, player\n",
      "   703, object\n",
      "   695, problem\n",
      "   695, believ\n",
      "   695, differ\n",
      "   689, inform\n",
      "   689, question\n",
      "   686, pleas\n",
      "   685, launch\n",
      "   681, comput\n",
      "   679, earth\n",
      "   674, realli\n",
      "   672, color\n",
      "   664, seem\n",
      "   663, line\n",
      "   659, number\n",
      "   651, still\n",
      "   648, someth\n",
      "   646, support\n",
      "   645, jesus\n",
      "   639, start\n",
      "   635, thank\n",
      "   632, might\n",
      "   631, gif\n",
      "   615, univers\n",
      "   613, nasa\n",
      "   611, list\n",
      "   605, play\n",
      "   603, sinc\n",
      "   601, part\n"
     ]
    }
   ],
   "source": [
    "# Look at some of the most frequent words\n",
    "tokens = []\n",
    "for pre in data.pre:\n",
    "    [tokens.append(w) for w in pre.split()]\n",
    "print(\"Total terms: {}\".format(len(tokens)))\n",
    "num_unique_terms = len(set(tokens))\n",
    "print(\"Unique terms: {}\".format(num_unique_terms))\n",
    "text = nltk.Text(tokens)\n",
    "fdist1 = nltk.probability.FreqDist(text)\n",
    "print(\"Term frequencies:\")\n",
    "for w in fdist1.most_common(100):\n",
    "    print(\"{:>6}, {}\".format(w[1], w[0].encode(\"utf-8\")))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build the TDM Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(max_df=0.6, max_features=10000,\n",
    "                             min_df=2, stop_words='english',\n",
    "                             use_idf=True)\n",
    "tdm = vectorizer.fit_transform(data.pre)\n",
    "\n",
    "# Get the vocabulary (i.e., terms mapped to their indices)\n",
    "vocab = vectorizer.vocabulary_\n",
    "\n",
    "# Inverse the vocabulary (i.e., indices mapped to terms)\n",
    "ivocab = {}\n",
    "for term, index in vocab.iteritems():\n",
    "    ivocab[index] = term"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run the clustering algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Try a few different values of K; we'll decide later what's the best.\n",
    "K = range(1,20)\n",
    "fits = [KMeans(n_clusters=k, init='k-means++', max_iter=500, n_init=30, verbose=False, random_state=1234).fit(tdm) for k in K]\n",
    "centroids = [fit.cluster_centers_ for fit in fits]\n",
    "inertias = [fit.inertia_ for fit in fits]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate and Inspect Clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs8AAAFwCAYAAABHBpaKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xuc1WW59/HPNaCoeChT1BkEcRSTcqu41UqrsdLU2oq2\nzQOglJWWorWtp6xwmKaDtd1ZorU7aKh49slDihgpuMstyKN4ChIYx0lmRCvFw3gC1v388VvTDDDI\nGmDmt2bm83691msWv/mtmWtsNXy9ve7rjpQSkiRJktavIu8CJEmSpN7C8CxJkiSVyPAsSZIklcjw\nLEmSJJXI8CxJkiSVyPAsSZIklWi94TkiBkXE3IiYHxGPR0Rt8XptRCyNiIeLjyOL14dHxGsdrv+s\nw9caHRGPRcSiiPhJ9/1YkiRJ0qY3cH03pJTejIjDUkqvRcQA4P6IuKv46R+nlH7cycuWpJRGd3L9\n58DpKaV5ETE9Ij6eUrp7I+qXJEmSekxJbRsppdeKTweRBe62k1ViHS9Z63pE7Axsk1KaV7x0FTCm\n9FIlSZKkfJUUniOiIiLmA8uAmR0C8NkR8UhE/Doi3tHhJbsVWzZmRcShxWtVwNIO9ywtXpMkSZJ6\nhVJXngsppf2BocBBETEK+Bmwe0ppP7JQ/V/F258FhhXbNs4Dro2IrTd96ZIkSVLPWm/Pc0cppZcj\nYjZw5Bq9zr8Cfle85y3greLzhyOiARgJNAO7dnjN0OK1tURE6uy6JEmStCmllNbVhtyp9YbniNgB\nWJFSeikitgQOBy6MiJ1TSsuKtx0PPNHh/hdSSoWI2B3YA3gqpbQ8Il6KiIOAecCpwCVv84N05eeQ\nesTkyZOZPHly3mVInfL9qXLle1PlKqJLuRkobeV5F+DKiKgga/O4IaU0PSKuioj9gALwNHBG8f4P\nAd+JiLeKnzsjpbS8+LmzgKnAFsD0lNKMLlcsSZIk5aSUUXWPA2uNnUspnbqO+38L/HYdn3sI2KeL\nNUqSJEllwRMGpS6oqanJuwRpnXx/qlz53lRfEuXYWxwRqRzrkiRJUt8REV3eMOjKsyRJklQiw7Mk\nSZJUIsOzJEmSVCLDsyRJklQiw7MkSZJUIsOzJEmSVCLDsyRJklQiw7MkSZJUIsOzJEmSVCLDsyRJ\nklQiw7MkSZJUIsOzJEmSVCLDsyRJklQiw7MkSZJUIsOzJEmSVCLDsyRJklQiw7MkSZJUIsOzJEmS\nVCLDsyRJklQiw7MkSZJUIsOzJEmSVCLDsyRJklQiw7MkSZJUIsOzJEmSVCLDsyRJklQiw7MkSZJU\nIsOzJEmSVCLDsyRJklQiw7MkSZJUIsOzJEmSVCLDsyRJklQiw7MkSZJUIsOzJEmSVCLDsyRJklQi\nw7MkSZJUIsOzJEmSVCLDsyRJklQiw7MkSZJUIsOzJEmSVCLDsyRJklSigXkXUG4aG5uYNGkqzc0F\nqqoqqK+fwIgRw/MuS5IkSWUgUkp517CWiEh51NXY2MThh0+hoaEOGAy0Ul1dy8yZEw3QkiRJfUxE\nkFKKrrzGto0OJk2a2iE4AwymoaGOSZOm5liVJEmSyoXhuYPm5gLtwbnNYFpaCnmUI0mSpDJjeO6g\nqqoCaF3jaiuVlf5jkiRJkuF5NfX1E6iurqU9QLcyYEAtp502Ib+iJEmSVDbcMLiGtmkbLS0FKisr\n2G+/CVx88XDuvRf22iuXkiRJktQNNmTDoOG5BL/5DVxwAcyaBXvskXc1kiRJ2hQ2JDw757kEn/kM\nrFwJH/kIzJ4Nu++ed0WSJEnKg+G5RJ//PKxa1R6gd9st74okSZLU0wzPXXDmmauvQA8blndFkiRJ\n6kmG5y46++zVA/TQoXlXJEmSpJ5ieN4AX/5y1sJx2GFw331QWZl3RZIkSeoJhucNdN557SvQs2bB\nLrvkXZEkSZK6m+F5I3z961mA/uhHswC90055VyRJkqTuZHjeSN/61uoBescd865IkiRJ3WW9x3NH\nxKCImBsR8yPi8YioLV6vjYilEfFw8XFkh9ecHxGLI2JhRBzR4froiHgsIhZFxE+650fqeRdcAMcd\nlwXov/8972okSZLUXUo6YTAitkopvRYRA4D7gXOAo4BXUko/XuPevYFrgQOBocAfgD1TSiki5gJn\np5TmRcR04Kcppbs7+X5ldcJgKVKCb34TZsyAe+6B7bfPuyJJkiS9nQ05YXC9K88AKaXXik8HkbV6\ntCXbzr7ZscD1KaWVKaWngcXAQRGxM7BNSmle8b6rgDFdKbacRcD3v5+tPh9+OLz4Yt4VSZIkaVMr\nKTxHREVEzAeWATM7BOCzI+KRiPh1RGxXvFYFPNPh5c3Fa1XA0g7Xlxav9RkR8J//CR/8IHz84/DS\nS3lXJEmSpE2p1JXnQkppf7I2jIMiYhTwM2D3lNJ+ZKH6v7qvzN4jAi6+GA46CI48El5+Oe+KJEmS\ntKl0adpGSunliJgNHLlGr/OvgN8VnzcDu3b43NDitXVd79TkyZP/+bympoaampqulJqrCJgyBb70\nJTj66KwPeuut865KkiSpf5s9ezazZ8/eqK+x3g2DEbEDsCKl9FJEbAncDVwIPJxSWla85yvAgSml\nU4qr0tcAB5O1ZcykfcPgHLLNhvOAO4FLUkozOvmevW7DYGcKBTjjDFi0CKZPh8GD865IkiRJbTZk\nw2Ap4Xkf4EqyFo8K4IaU0vci4ipgP6AAPA2ckVJ6rvia84HTgRXAuSml3xevHwBMBbYApqeUzl3H\n9+wT4RmyAH366dDUBHfcAVttlXdFkiRJgm4Kz3noS+EZYNUq+Mxn4Nln4fbbYcst865IkiRJ3Taq\nThtnwAD4zW9gyJDsMJU33si7IkmSJG0Iw3MPGTAArrwSttsOPvUpePPNvCuSJElSVxmee9DAgTBt\nWta2ccIJ8NZbeVckSZKkrjA897DNNoPrrstWok88EVasyLsiSZIklcrwnIPNNoMbbsg2Ep58sgFa\nkiSptzA852TzzeGmm+D112H8eFi5Mu+KJEmStD6G5xwNGgT/9//Ciy/CaadlK9GSJEkqX4bnnG2x\nBdx6Kzz3XDYL2gAtSZJUvgzPZWDLLbPDU555Bj7/+exUQkmSJJUfTxgsI6++CkcfDe9+N3z9603U\n1k6lublAVVUF9fUTGDFieN4lSpIk9Rkez90HvPIK1NQ0sWTJFF5+uQ4YDLRSXV3LzJkTDdCSJEmb\niMdz9wHbbAN77DG1Q3AGGExDQx2TJk3NsTJJkiQZnsvQ888XaA/ObQbT0mIztCRJUp4Mz2WoqqoC\naF3jaiuVlf7PJUmSlCfTWBmqr59AdXUt7QG6lQEDajnhhAn5FSVJkiQ3DJarxsYmJk2aSktLgcrK\nCg49dAKTJg3n0kvhxBPzrk6SJKn3c9pGH/fII3DssdlphJMnQ4X/3UCSJGmDGZ77geeeg+OOg8pK\nuPJKGLzmvkJJkiSVxFF1/cBOO8GsWVlo/uAHs1MJJUmS1DMMz73QoEEwdSqcfDK8730wZ07eFUmS\nJPUPtm30cnfcAZ/5DPz4xzB+fN7VSJIk9R72PPdTTzwBxxwDn/40fP/7biSUJEkqheG5H/v73+FT\nn4J3vAOmTcuO+ZYkSdK6uWGwH9thB5g5E4YMgUMOgaefzrsiSZKkvsfw3Idsvjn88pdw+unw/vfD\nn/6Ud0WSJEl9i+G5j4mAc8/NpnEcfzxccUXeFUmSJPUd9jz3YX/5C/zbv2WbCX/0IxgwIO+KJEmS\nyocbBrWWF17IpnBsthlcfz1st13eFUmSJJUHNwxqLdtvD3fdBbvvnvVBL1mSd0WSJEm9l+G5H9hs\nM7jsMjj77GwSx6xZeVckSZLUOxme+5EvfQmuuw5OOgl+8Yu8q5EkSep97Hnuh5YsyTYSfuxjcPHF\nMHBg3hVJkiT1PDcMqmTLl2cr0KtWwY03wjvfmXdFkiRJPcsNgyrZO94Bd9wB++wDBx8MTz6Zd0WS\nJEnlz/Dcjw0cCD/+MXz96/DBD8Lvf593RZIkSeXN8CxOPx1uvhlOPRUuuQTsmJEkSeqcPc/6p8bG\nbCPhIYfAlCmw+eZ5VyRJktR93DCojfbyyzB2LLzyCvzXfzVx8cVTaW4uUFVVQX39BEaMGJ53iZIk\nSZuE4VmbxKpVcNZZTVx++RRWrqwDBgOtVFfXMnPmRAO0JEnqE5y2oU1iwAB49dWpHYIzwGAaGuqY\nNGlqjpVJkiTly/CsTjU3F2gPzm0G09JSyKMcSZKksmB4VqeqqiqA1jWutuJbRpIk9WcmIXWqvn4C\n1dW1tAfoVnbZpZaFCydwxhnZhkJJkqT+xg2DWqfGxiYmTZpKS0uBysps2sb22w/nP/4D7r0XLr8c\nPvKRvKuUJEnaME7bUI+ZPh2+8AUYMwYuvBC23jrviiRJkrrGaRvqMUcfDY8/ns2F3ndf+OMf865I\nkiSp+7nyrI12223wxS/CiSfC974HW22Vd0WSJEnr58qzcnHssdkq9LJlsP/+8MADeVckSZLUPVx5\n1iZ1881w9tlw2mlQVwdbbJF3RZIkSZ1z5Vm5+/d/h8ceg4YGGD0a5s3LuyJJkqRNx/CsTW7IELjp\nJrjgAvjkJ+Fb34I338y7KkmSpI1neFa3iICTToJHH4UnnoADD4T58/OuSpIkaeMYntWtdt4Zbr0V\nvvY1+PjHsz7oFSvyrkqSJGnDGJ7V7SJg/Phs5XnuXDj44KwvWpIkqbcxPKvHVFXBnXdm0zg++tFs\nJvTKlXlXJUmSVDpH1SkXf/0rnH46LF8OV14Jo0blXZEkSepvHFWnXmPYMPj97+Fzn4MPfxj+8z9h\n1aq8q5IkSXp7rjwrd42N8NnPZuPsfvMb2GuvvCuSJEn9QbesPEfEoIiYGxHzI+LxiKhd4/PnRUQh\nIrYv/nl4RLwWEQ8XHz/rcO/oiHgsIhZFxE+6Uqj6rhEj4J574JRT4JBD4OKLoVDIuypJkqS1lbTy\nHBFbpZRei4gBwP3AOSmlByNiKPBrYC/ggJTSCxExHPhdSulfOvk6c4GzU0rzImI68NOU0t2d3OfK\ncz+1ZAlMmAAVFdkqdHV13hVJkqS+qtt6nlNKrxWfDgIGAm3J9mLga53V0klxOwPbpJTaDmy+ChjT\nlWLV9+2xB9x3Hxx3XDbS7rLLoKGhiXHj6jjssFrGjaujsbEp7zIlSVI/NbCUmyKiAngIqAYuK64c\nHwM8k1J6PGKtrLxbRDwMvARMSin9CagClna4Z2nxmrSaAQPgK1+Bo4+Gk05q4mtfm8Lrr9cBg4FW\n5sypZebMiYwYMTzvUiVJUj9T6spzIaW0PzAUOCgi9gG+CXTsf25L0C3AsJTSaOA84NqI2HoT1qx+\nYq+9YO+9p3YIzgCDaWioY9KkqTlWJkmS+quSVp7bpJRejojZwLHAbsCjkS07DwUeioiDUkrPAy8W\n7384IhqAkUAzsGuHLze0eK1TkydP/ufzmpoaampqulKq+ohnny3QHpzbDGbpUncUSpKkrpk9ezaz\nZ8/eqK+x3g2DEbEDsCKl9FJEbAncDVyYUpre4Z5GYHRK6cXi/S+klAoRsTtwH7BPSml5RMwBzgHm\nAXcCl6SUZnTyPd0wKADGjavjmmu+yuoBupVttrmIyy+v5d//PTv+W5Ikqau6a8PgLsCsiHgEmAvc\n3TE4FyXa2zY+BDxW7Hm+ETgjpbS8+LmzgMuBRcDizoKz1FF9/QSqq2uB1uKVVqqra7n00gn84AfZ\npsJZs3IsUJIk9SsekqKy19jYxKRJU2lpKVBZWUF9/QRGjBhOoQA33gjf+haMHAkXXgj77pt3tZIk\nqbfYkJVnw7N6vbfegl/+Er77XfjYx6C+Pjt4RZIk6e1025xnqZxtvjmcfTYsXpzNif7Xf4Uvfxn+\n9re8K5MkSX2N4Vl9xjbbwOTJsGABrFoFe++drUa3tq73pZIkSSUxPKvP2WknmDIF5s7NgvSee8LP\nfw4rVuRdmSRJ6u0Mz+qzqqvh2mvhjjvglltg1Khsg6Ht9JIkaUO5YVD9xsyZ8I1vQEUF/PCH8JGP\n5F2RJEnKk9M2pPUoFOCmm7LxdnvskY2322+/vKuSJEl5cNqGtB4VFXDiiVkv9L/9Gxx1FIwdC089\nlXdlkiSpNzA8q1/afHM466xsvN1ee8GBB8K55zreTpIkvT3Ds/q1rbeGCy6AhQuzjYR7750dsvLq\nq3lXJkmSypHhWQKGDIFLLoEHH4S//CUbb/eznzneTpIkrc4Ng1In5s/PJnM89RR873twwAFN1NZO\npbm5QFVVBfX1ExgxYnjeZUqSpI3gtA1pE7vnHjj33CaWLJnCm2/WAYOBVqqra5k5c6IBWpKkXsxp\nG9Im9tGPwr77Tu0QnAEG09BQx6RJU3OsTJIk5cHwLK1HS0uB9uDcZjAPPFDgH//IoyJJkpQXw7O0\nHlVVFUDrGldbSamCPfaAz30OHn00j8okSVJPMzxL61FfP4Hq6lraA3TW83zPPRN48kkYMQI+8Qn4\n0Iey0wtXrsyxWEmS1K3cMCiVoLGxiUmTptLSUqCycu1pGytWwK23wpQp0NgIX/wifP7zsOOO+dUs\nSZLentM2pDLwyCNZiP7tb+HYY2HiRDjggLyrkiRJa3LahlQG9tsPLr8clizJTiw87jg45BC4/noP\nXZEkqbdz5VnqZitXwu23Z6vRixbBmWfCF74AO+2Ud2WSJPVvrjxLZWjgQDj+eJg1C2bMgKVL4d3v\nhvHjs+PAJUlS7+HKs5SDF16AK66Ayy6DIUPgnHPghBNg883zrkySpP7DDYNSL7NqFdxxR9bS8ec/\nZ+0cZ54Ju+ySd2WSJPV9tm1IvcyAAdlEjj/8IXs8/zyMGgWnnAIPPAD+O6QkSeXFlWepzCxfDr/5\nDVx6KbzzndmouxNPhC22aJ833dxcoKpq7XnTkiSpdLZtSH1IoQB33QWXXJLNjj7hhCbuuGMKTU11\nwGDaTjqcOXOiAVqSpA1geJb6qCefhGOOqWPRoq+SBec2rYwdexHTptXmVZokSb2WPc9SH7XXXlBZ\nWWD14AwwmEWLCnmUJElSv2R4lnqJqqoKoHWNq6088UQFBxyQtXf87W95VCZJUv9heJZ6ifr6CVRX\n19IeoLOe58cem8CFF2YHruy5Zza947e/hTffzLFYSZL6KHuepV6kbdpGS0uBysq1p2288grcfDNc\neSU88UQ2peO00+DAAyG61NElSVLf54ZBSf/09NNw9dVw1VXZEeGnngrjxsGuu+ZdmSRJ5cHwLGkt\nKWUHrlx5Jdx0E4wenQXp44+HrbfOuzpJkvJjeJb0tt54A26/PVuNvv/+rD/61FOhpgYq3AEhSepn\nDM+SSrZsGVx7bbYivXw5jB+fBemRI/OuTJKknmF4lrRBHn00C9HXXgsjRmQh+sQTYfvt865MkqTu\nY3iWtFFWroS7787aOmbMgCOOyIL0kUfCZptl97RN/GhuLlBVtfbED0mSegvDs6RN5sUX4cYbsyC9\nZAmcfDJ87GNNfPnLU2hoqCM77TCbNT1z5kQDtCSp1zE8S+oWixdnIfrii+tobf0qqx8T3srYsRcx\nbVptXuVJkrRBNiQ8u79e0nrtuSfU18OBBxZYPTgDDOaZZwp5lCVJUo8zPEsqWVVVBe3Hg7dp5YEH\nKhg3LuuTXrkyj8okSeoZhmdJJauvn0B1dS3tATrreb7//gkcfDBccAEMHQpf+Qo89FB2QIskSX2J\nPc+SuqRt2kZLS4HKyrWnbTz5JFxzDUybBltskR0JPnYsDHc/oSSpzLhhUFLZSAn+93+zEH3TTfCe\n92RB+oQT4B3vyLs6SZIMz5LK1Jtvwl13ZUF65kw4/PDsRMOjjoLNN8+7OklSf2V4llT2XnwRbr4Z\nrr4aFizIVqLHj4f3vx+iS7++JEnaOIZnSb3K009nR4JffTW89VbWGz1uHIwcmXdlkqT+wPAsqVdK\nCR5+OAvR11+fbS4cPx5OPBF23DHv6iRJfZXhWVKvt3Il/OEPWZC+80744Aez1ehjjoEtt2y/r23q\nR3Nzgaqqtad+SJK0PoZnSX3KK6/ArbdmQXrePBgzJluRHjasiSOPnEJDQx3ZiYfZvOmZMycaoCVJ\nJTM8S+qzWlrguuuyiR1PPlnH669/ldWPCm9l7NiLmDatNq8SJUm9zIaEZ08YlNQrVFbCeefB/Pnw\n3vcWWD04AwymubmQR2mSpH7E8Cyp1xk5soL2I8LbtDJnTgVnngkzZmSzpSVJ2tQMz5J6nfr6CVRX\n19IeoLOe5zvvnEB1NdTXw847w8knww03wMsv51isJKlPsedZUq/UNm2jpaVAZeXa0zaWLYPbb4db\nboH774dDD4Xjjsumduy0U351S5LKhxsGJakTL78M06dnkztmzID3vjeb3DFmDOyxR97VSZLyYniW\npPV48024995sRfr227NDWMaMyVal99/fI8IlqT/plmkbETEoIuZGxPyIeDwiatf4/HkRUYiI7Ttc\nOz8iFkfEwog4osP10RHxWEQsioifdKVQSdoUBg2Co46CX/4SmpvhF7+AN96AT38adtsNzj0XZs3K\nDmuRJGlNJa08R8RWKaXXImIAcD9wTkrpwYgYCvwa2As4IKX0QkTsDVwLHAgMBf4A7JlSShExFzg7\npTQvIqYDP00p3d3J93PlWVKPSgn+/OesteOWW6CpCT75yWxV+ogjYKut8q5QkrSpdduc55TSa8Wn\ng4CBQFuyvRj42hq3Hwtcn1JamVJ6GlgMHBQROwPbpJTmFe+7ChjTlWIlqbtEZL3Q3/42PPQQPPww\njB4Nl1ySTe447ji46ip44YX21zQ2NjFuXB2HHVbLuHF1NDY25fcDSJJ6xMBSboqICuAhoBq4rLhy\nfAzwTErp8Vi9SbAKeKDDn5uL11YCSztcX1q8LkllZ9gwOOec7PGPf8Add2Qr0mefDQceCIce2sSV\nV06hqan9iPA5czwiXJL6ulJXngsppf3J2jAOioh9gG8CnoMrqc9717vgtNOy8LxsGUycCFdfPbVD\ncAYYTENDHZMmTc2xUklSdytp5blNSunliJhN1pqxG/BoZMvOQ4GHI+IgspXmYR1eNrR4rRnYtZPr\nnZo8efI/n9fU1FBTU9OVUiWpW2y1VdYH/dOfFmhsXPuI8AULCqTk1A5JKkezZ89m9uzZG/U11rth\nMCJ2AFaklF6KiC2Bu4ELU0rTO9zTCIxOKb0YEaOAa4CDydoyZtK+YXAOcA4wD7gTuCSlNKOT7+mG\nQUllbdy4Oq655qu0rzwDtLLNNhcxdGgtZ54J48fDO9+ZV4WSpPXprg2DuwCzIuIRYC5wd8fgXJSA\nAEgpLQBuBBYA04EvdUjCZwGXA4uAxZ0FZ0nqDdZ1RPgjj0zg5z+HBx6AESPgM5+BOXOyaR6SpN7P\nQ1IkaQOt74jw55+HqVOzmdJbbw1nnAFjx8K22+ZWsiSpA08YlKQyVCjAPfdkB7Lccw+ccAKceWY2\nCk+SlB/DsySVuWefhSuuyFajhwzJQvRJJ8HgNfceSpK6neFZknqJVatgxoxsNfpPf4JTTsnaOvbZ\nJ+/KJKn/6LYTBiVJm9aAAfCJT8Dtt8Ojj2azpI88Eg45BK6+Gl5/Pe8KJUmdceVZksrEypXZSYb/\n/d/ZEeHjx2er0XvtlXdlktQ3ufIsSb3YwIHZASwzZsDcuTBoEHz4w3DYYXDDDfDWW3lXKEly5VmS\nythbb8Gtt2a90U88kc2N/sIXYPfd2+9pG5nX3FygqmrtkXmSpM65YVCS+rBFi7IpHVdemY25O+MM\neO97mzj66Ck0NNSRnXaYHdYyc+ZEA7QkrYfhWZL6gTfegJtvzlaj/9//q+ONN9Y+Jnzs2IuYNq02\nrxIlqVew51mS+oEttoBx4+CPf4R99imwenAGGExLSyGP0iSpzzM8S1IvNnJkBdC6xtVWtt/eX++S\n1B387SpJvVh9/QSqq2tpD9CtbLttLTNnTuCCC+Cll3IsTpL6IHueJamXa5u20dJSoLIym7YBw5k8\nGe66C/7P/4GzzoItt8y3TkkqN24YlCSt5s9/hkmT4MEHs4+f/SxstlneVUlSeXDDoCRpNe95D/z2\nt3DLLdmEjr33hmuvhYL7CSVpg7jyLEn9yL33wje/Ca+9Bt/7HnzykxBdWnORpL7Dtg1J0nqlBL/7\nHXzrW7DNNvD970NNTd5VSVLPMzxLkkq2ahVcfz1ccAFUV2ch+l//Ne+qJKnn2PMsSSrZgAEwdiws\nXAjHHw/HHguf+lT2Z0lS5wzPktTPbb45nHkmLF4MBx8MH/oQTJgATz+dd2WSVH4Mz5IkALbaKpsJ\nvWQJDBsGBxwA55wDy5blXZkklQ/DsyRpNdttB9/5Tta+MWBANu7um9+EF1/MuzJJyp/hWZLUqSFD\n4OKLYf58eP55GDkSfvADaG1d/2slqa8yPEuS3tawYfDrX8Of/pQF6T33hMsug7feyrsySep5jqqT\nJHXJww/Dt7+dtXXU1WUTO/761yYmTZpKc3OBqqoK6usnMGLE8LxLlaS35ZxnSVKP+eMf4fzz4bnn\nmnjllSk891wdMBhopbq6lpkzJ/ZogG5sNMBL6hrDsySpR6UEH/lIHbNnf5UsOLdp5dBDL+L882vZ\naisYPDh7dHy+5Zab7mjwxsYmDj98Cg0N+QZ4Sb3LhoTngd1VjCSp78vCb4HVgzPAYJ58ssCll2Yb\nDF97LfvY8fkbb2QBes1Q3fZ8XaG7s+cXXji1Q3DOvn9DQx2TJl3EtGm1PfmPRFIfZ3iWJG2UqqoK\noJU1V56POKKCadPW/bpCAV5/fe1Q3fH5mteef77z6w891HmAb2kpbPKfV1L/ZniWJG2U+voJzJlT\nu1bLRH39xLd9XUVF+wryxho3roJrrlk7wL/5ZgUpbbr2EEmy51mStNHaNuu1tBSorOz5zXqd9TxX\nVdWy7bYT2Wmn4Vx6aXbYiyR15IZBSVK/1VmAHzZsOL/4BUyeDOPHQ20tbLtt3pVKKheGZ0mSOvH8\n89lYvRkz4Ec/glNOsZVDkuFZkqS3NWcOnHUWbL01XHop7LNP3hVJytOGhGeP55Yk9Rvvex88+CCc\ndBJ89KNi8AYSAAAP8klEQVTw5S/DSy/lXZWk3sTwLEnqVwYMgC9+ERYsyEbd7b03XHVVduCLJK2P\nbRuSpH7twQezVo5Bg+Cyy2DfffOuSFJPsW1DkqQuOuigrBd6/Hg44gg45xxYvjzvqiSVK8OzJKnf\nGzAAzjgja+V4882slWPq1OwUREnqyLYNSZLWMG9e1soxcGDWyrH//nlXJKk72LYhSdImcOCBWSvH\nZz8LRx6ZBekXX8y7KknlwPAsSVInKirgc5+DhQuz9o2994YrrrCVQ+rvbNuQJKkEDz2UrUBD1spx\nwAH51iNp49m2IUlSNzngAPjf/4XPfx4+8YlsVvQLL+RdlaSeZniWJKlEFRVw+unZVI6KiqyV41e/\nspVD6k9s25AkaQPNn5+1cqxcmbVyHHggNDY2MWnSVJqbC1RVVVBfP4ERI4bnXaqkTmxI24bhWZKk\njVAoZMd7n38+1NQ0MWfOFJ5+ug4YDLRSXV3LzJkTDdBSGTI8S5KUk+XL4eCD61i06KtkwblNK2PH\nXsS0abV5lSZpHdwwKElSTt7xDqisLLB6cAYYzOOPF5wTLfURA/MuQJKkvqKqqgJoZc2V52XLKhg2\nDIYOhfe/H973vuzjqFHZ0eCSeg/bNiRJ2kQaG5s4/PApNDSs3fO8667DeeIJeOCB7PTCBx6A557L\nNhm2hemDD4Yddsj7p5D6D3ueJUnKWdu0jZaWApWVbz9t4+9/h7lz28P0gw/Czju3h+n3vQ/22QcG\n+t+JpW5heJYkqRdbtSqbId0WpufMgWeeyQ5o6djuMWRI3pVKfYPhWZKkPubFF7MV6bYwPXcubL/9\n6qvT++4Lm23W/hpnTUulMTxLktTHFQrwl7+svjrd2Aj775+F6REjmvjhD6fQ1OSsaWl9DM+SJPVD\nL70E8+ZlYfq//7uOlhZnTUulcM6zJEn90Hbbwcc+BpMmwciRnc+avuuuAr/8ZTbhQ9KGMzxLktSH\ntM+a7qiVUaMqmDUL9toLDjkELroIGhryqFDq3dYbniNiUETMjYj5EfF4RNQWr38nIh4tXp8RETsX\nrw+PiNci4uHi42cdvtboiHgsIhZFxE+678eSJKl/qq+fQHV1Le0BOut5vuqqCVx3Xbby/O1vw6JF\n8IEPwL/8C9TWwvz5YMektH4l9TxHxFYppdciYgBwP3AOsCCl9Grx8xOBUSmlL0bEcOB3KaV/6eTr\nzAXOTinNi4jpwE9TSnd3cp89z5IkbaBSZ02vWpVtOLzlluxRKMCYMXDccdnqtKcfqq/r9g2DEbEV\n8D/AF1NK8zpc/wawa0rprGJ4viOltM8ar90ZuDelNKr455OAD6eUvtjJ9zE8S5LUg1KCxx9vD9It\nLXDMMVmQ/uhHYYst8q5Q2vS6bcNgRFRExHxgGTCzLThHxHcj4q/AKcAFHV6yW7FlY1ZEHFq8VgUs\n7XDP0uI1SZKUs4j2Fo5HHsnmSb/nPfDDH2anHn7603DdddlkD6k/Kyk8p5QKKaX9gaHAwRExqnj9\n2ymlYcA1wMTi7c8Cw1JKo4HzgGsjYutNX7okSeouI0bAV74C//M/WX/0EUfANdfArrvCUUfBL34B\ny5blXaXU8wZ25eaU0ssRMQs4EljQ4VPXAtOBySmlt4C3ivc/HBENwEigGdi1w2uGFq91avLkyf98\nXlNTQ01NTVdKlSRJm8iQIfC5z2WPV16Bu+7KWju+8Q0YNSpr7TjuOKiuXv11nnSocjN79mxmz569\nUV9jvT3PEbEDsCKl9FJEbAncDVwILEopLSneMxH4YErp08X7X0gpFSJid+A+YJ+U0vKImEO22XAe\ncCdwSUppRiff055nSZLK3Jtvwr33wq23wm23wY47tgfp7bZr4ogjptDQ4EmHKl/dsmEwIvYBriRr\n8agAbkgpfS8ibiZbUS4ATcCZKaVnI+J44Dtkq88F4IKU0vTi1zoAmApsAUxPKZ27ju9peJYkqRdZ\nc3LHc8/V0drqSYcqbx7PLUmScpcSHHxwLfPm1a31ud12q+XCC+sYNQpGjoRBg3IoUCrakPDcpZ5n\nSZKk9YmAkSMrmDevlTVXnrfeuoIbb4QFC6CxEXbbLeubfs97so+jRmWnIDoaT+XKlWdJkrTJNTY2\ncfjhb9/z/NZb2SSPBQuyx5//nH186qlsqkdbmG4L1u9+N2y5Za4/lvoY2zYkSVLZKPWkwzWtWAGL\nF68dqpcsgaqqzkP14MGdfy0nfujtGJ4lSVKftWIFNDSsHaoXLYJddmkP1W3BesstmxgzxokfWjfD\nsyRJ6ndWrsxaPdYM1Y8/XseqVU780Lq5YVCSJPU7AwdmkztGjoQxY9qv19QUuO++Nfs5BjNnToGW\nFqis7NEy1UeUdDy3JElSbzN0aAXQusbVVlKq4L3vzY4cv/pqePXVPKpTb2V4liRJfVJ9/QSqq2tp\nD9BZz/Mf/jCB5mY4/XS4/noYOhROPRVmzswOe5Hejj3PkiSpzypl4sdzz2Uh+uqr4dln4ZRTsjC9\nzz751Kye44ZBSZKkjbBgQRaip02Dd70Lxo/PwvQuu+RdmbqD4VmSJGkTKBTgvvvgqqvg1lvh4IOz\nID1mzLpnSqv3MTxLkiRtYq+9Brfdlq1IP/AAHHtsFqRramDAgLyr08YwPEuSJHWjZcvguuuyIP38\n8zB2bBak3/vevCvThjA8S5Ik9ZAnnshC9DXXwJAhWYg++WTYeee8K1OpDM+SJEk9bNUqmD07C9K3\n3gof+EAWpI89FrbaKrunbepHc3OBqqrOp36o5xmeJUmSctTamgXoq6+GuXOzDYaHH97EpElTeOqp\nOrKjwrN50zNnTjRA58zwLEmSVCaefRauvRa++906li//KllwbtPK2LEXMW1abV7liQ0Lz54wKEmS\n1A122QXOOw/226/A6sEZYDC33Vbg1FPhBz+AW26Bv/wFVqzIo1J1xcC8C5AkSerLqqoqyI4IX33l\n+QMfqKCmBhYuhCuuyD4uXQojRsDee2ePd7+7/ePWW+dTv1Zn24YkSVI3amxs4vDDp9DQsP6e5zfe\ngMWLsyC9cGG2Gr1wISxaBDvs0B6mOwbrIUMgSmg8cNPi2ux5liRJKkNtwbWlpUBlZdeD66pV8Ne/\nrh2qFy6ElDoP1bvt1n6IS1cCfH9ieJYkSepHUoK//73zUP23v8Eee2RBeuHCOh5/3E2La9qQ8GzP\nsyRJUi8VATvumD0+9KHVP9faCk8+mQXpuXM737Q4a1aBn/wERo7MHrvtBgNNh2/LfzySJEl90ODB\nMHp09rjrrgqamtbetLjLLhU0NMBdd2V91c8+C8OHZ0F6zz3bQ/XIkVBZCRXOabNtQ5Ikqa8rtef5\njTfgqaeyIL14cfax7flLL2VtIGsG6z33zDYzlrJpsa2Wctm4aM+zJEmSOrWxmxZfeSUL0WuG6ief\nzD7fWajec0/YdtvVayinjYuGZ0mSJPWolOAf/1g7VLd93Hbb9lD90EN1PPJI+WxcdMOgJEmSelRE\n1raxww7w/vev/rmUoKWlPVT//vedb1xsaSn0VLkbzbZvSZIkdYsIqKqCww6DM86AD32o7bTFjlqp\nrOw9kbT3VCpJkqRerb5+AtXVtbQH6Kznub5+Qm41dZU9z5IkSeoxG7txcVNyw6AkSZJUog0Jz7Zt\nSJIkSSUyPEuSJEklMjxLkiRJJTI8S5IkSSUyPEuSJEklMjxLkiRJJTI8S5IkSSUyPEuSJEklMjxL\nkiRJJTI8S5IkSSUyPEuSJEklMjxLkiRJJTI8S5IkSSUyPEuSJEklMjxLkiRJJTI8S5IkSSUyPEuS\nJEklMjxLkiRJJTI8S5IkSSUyPEuSJEklMjxLkiRJJTI8S5IkSSUyPEuSJEklMjxLkiRJJTI8S5Ik\nSSUyPEuSJEklWm94johBETE3IuZHxOMRUVu8/p2IeLR4fUZE7NzhNedHxOKIWBgRR3S4PjoiHouI\nRRHxk+75kSRJkqTusd7wnFJ6EzgspbQ/sB9wVEQcBPwopbRv8fqdQFuoHgV8GtgbOAr4WURE8cv9\nHDg9pTQSGBkRH9/kP5HUjWbPnp13CdI6+f5UufK9qb6kpLaNlNJrxaeDgIHZpfRqh1sGA4Xi82OA\n61NKK1NKTwOLgYOKK9PbpJTmFe+7ChizkfVLPcq/AFTOfH+qXPneVF8ysJSbIqICeAioBi5rC8AR\n8V3gVGA5cFjx9irggQ4vby5eWwks7XB9afG6JEmS1CuUuvJcKLZnDAUOLrZmkFL6dkppGHANMLH7\nypQkSZLyFymlrr0gYhLQmlL6cYdruwJ3ppT+JSK+QdbW8cPi52aQ9UM3AbNSSnsXr58EfDil9MVO\nvkfXipIkSZI2QEop1n9Xu/W2bUTEDsCKlNJLEbElcDhwYUTskVJaUrxtDPCX4vPbgWsi4mKytow9\ngAdTSikiXipuNpxH1u5xyab4ISRJkqSeUErP8y7AlcW+5wrghpTS9Ii4OSJGkm0UbALOBEgpLYiI\nG4EFwArgS6l9efssYCqwBTA9pTRjk/40kiRJUjfqctuGJEmS1F+V1QmDEXFkRPyleIjK1/OuR+oo\nIp7ucDDQg3nXo/4rIi6PiOci4rEO194ZEb+PiCcj4u6I2C7PGtV/reP9WRsRSyPi4eLjyDxrVP8U\nEUMj4t6I+HPx4L9zite79PuzbMJzsS3kUuDjwHuAkyPi3flWJa2mANSklPZPKR2UdzHq135D9ruy\no28Af0gp7QXcC5zf41VJmc7enwA/TimNLj5s21QeVgL/kVJ6D/B+4Kxi1uzS78+yCc/AQcDilFJT\nSmkFcD1wbM41SR0F5fX/GfVTKaU/AS+ucflY4Mri8yvxECrlZB3vT8h+h0q5SSktSyk9Unz+KrCQ\nbAxzl35/llMQqAKe6fBnD1FRuUnAzIiYFxGfz7sYaQ1DUkrPQfYXBDAk53qkNZ0dEY9ExK9tK1Le\nImI3YD9gDrBTV35/llN4lsrdISml0cDRZP+p59C8C5LehrvBVU5+BuyeUtoPWAb8eD33S90mIrYG\nbgbOLa5Ar/n78m1/f5ZTeG4GhnX489DiNakspJSeLX78G3ALWauRVC6ei4idACJiZ+D5nOuR/iml\n9LcOY2t/BRyYZz3qvyJiIFlwvjqldFvxcpd+f5ZTeJ4H7BERwyNic+AksgNXpNxFxFbFf1MlIgYD\nRwBP5FuV+rlg9R7S24EJxeenAbet+QKpB632/iwGkjbH4+9P5ecKYEFK6acdrnXp92dZzXkujq75\nKVmovzyldGHOJUkARMQIstXmRHa40DW+P5WXiLgWqAHeBTwH1AK3AjcBu5IdXPXplNLyvGpU/7WO\n9+dhZP2lBeBp4Iy2HlOpp0TEIcD/AI+T/X2egG8CDwI3UuLvz7IKz5IkSVI5K6e2DUmSJKmsGZ4l\nSZKkEhmeJUmSpBIZniVJkqQSGZ4lSZKkEhmeJUmSpBIZniVJkqQSGZ4lSZKkEv1/L9tpmavnLa0A\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12a41f310>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "clusters_df = pd.DataFrame( { \"num_clusters\":K, \"cluster_errors\": inertias } )\n",
    "plt.figure(figsize=(12,6))\n",
    "plt.plot( clusters_df.num_clusters, clusters_df.cluster_errors, marker = \"o\" )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I was hoping to see a elbow in the curve, but it looks like the metric just gets lower and lower as the number of clusters increases. \n",
    "\n",
    "### Examine Top Cluster Terms\n",
    "\n",
    "For no real good reason, let's select `k=10` and examine some of the resulting clusters.\n",
    "\n",
    "First, let's print out the top words/terms in each cluster. This will help us to remember them better."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster  0 ( 163 docs): file format gif convert cview program ftp graphic pov\n",
      "Cluster  1 (  71 docs): moral object valu say christian kuiper smiley subject peopl\n",
      "Cluster  2 ( 323 docs): game team win pitch run score brave pitcher basebal\n",
      "Cluster  3 (1131 docs): like know think don time peopl new say right\n",
      "Cluster  4 ( 646 docs): graphic thank mail pleas look use program polygon help\n",
      "Cluster  5 ( 194 docs): orbit mission hst solar shuttl satellit spacecraft comet jupit\n",
      "Cluster  6 ( 277 docs): hit year player bat hitter ball defens good play\n",
      "Cluster  7 (  37 docs): jewish player basebal lowenstein come hank koufax stankowitz sandi\n",
      "Cluster  8 ( 216 docs): god christian jesus bibl christ believ peopl love law\n",
      "Cluster  9 ( 392 docs): space moon launch nasa cost station shuttl billion sky\n"
     ]
    }
   ],
   "source": [
    "# The top terms for each cluster\n",
    "\n",
    "k = 10 # Let's select k = 10, just for fun.\n",
    "km = fits[k]\n",
    "data.assigned_cluster = km.labels_\n",
    "\n",
    "cluster_terms = []\n",
    "order_centroids = km.cluster_centers_.argsort()[:, ::-1]\n",
    "terms = vectorizer.get_feature_names()\n",
    "for i in range(k):\n",
    "    print(\"Cluster {:>2} ({:>4} docs):\".format(i, len([x for x in km.labels_ if x == i])), end='')\n",
    "    cterms = []\n",
    "    for ind in order_centroids[i, :9]:\n",
    "        print(' {}'.format(terms[ind]), end='')\n",
    "        cterms.append(terms[ind])\n",
    "    cluster_terms.append(' '.join(cterms))\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### \"Confusion Matrix\"\n",
    "\n",
    "Since we have the \"truth\" categories from which each document came, we can build a \"confusion-matrix\"-like table that shows, for each cluster, which categories the assigned documents came from."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix:\n",
      "\n",
      "[[161   3   0 120 572   0   0   0   0   1]\n",
      " [  2   0 323 320  35   0 277  37   0   0]\n",
      " [  0   9   0 343  34 194   0   0   0 391]\n",
      " [  0  59   0 348   5   0   0   0 216   0]]\n",
      "\n",
      "Rows:\n",
      " 0: comp.graphics\n",
      " 1: rec.sport.baseball\n",
      " 2: sci.space\n",
      " 3: talk.religion.misc\n",
      "\n",
      "Columns:\n",
      " 0: file format gif convert cview program ftp graphic pov\n",
      " 1: moral object valu say christian kuiper smiley subject peopl\n",
      " 2: game team win pitch run score brave pitcher basebal\n",
      " 3: like know think don time peopl new say right\n",
      " 4: graphic thank mail pleas look use program polygon help\n",
      " 5: orbit mission hst solar shuttl satellit spacecraft comet jupit\n",
      " 6: hit year player bat hitter ball defens good play\n",
      " 7: jewish player basebal lowenstein come hank koufax stankowitz sandi\n",
      " 8: god christian jesus bibl christ believ peopl love law\n",
      " 9: space moon launch nasa cost station shuttl billion sky\n"
     ]
    }
   ],
   "source": [
    "# Rows are the actual categories, colums are the assigned clusters\n",
    "print(\"Confusion matrix:\\n\")\n",
    "cmat = metrics.confusion_matrix(y_true=data.target, y_pred=km.labels_)\n",
    "print(cmat[0:num_categories, 0:k])\n",
    "print(\"\\nRows:\")\n",
    "for i, name in enumerate(data.target_names):\n",
    "    print(\" {}: {}\".format(i, name))\n",
    "print(\"\\nColumns:\")\n",
    "for i, terms in enumerate(cluster_terms):\n",
    "    print(\" {}: {}\".format(i, terms))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pretty neat! "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Look at some examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def printDoc(j, data, km, cluster_terms):\n",
    "    '''\n",
    "    Given a document ID, print everything we know about it.\n",
    "    '''\n",
    "    print(\"Document ID: {}\".format(j))\n",
    "    print(\"Filename: {}\".format(os.path.basename(data.filenames[j])))\n",
    "    print(\"Truth category: {} ({})\".format(data.target[j], data.target_names[data.target[j]]))\n",
    "    print(\"Assigned cluster: {} ({})\".format(km.labels_[j], cluster_terms[km.labels_[j]]))\n",
    "    print(\"Raw data:\")\n",
    "    print(\"============ START RAW ==========================\\n\")\n",
    "    print(data.data[j].encode(\"utf-8\"))\n",
    "    print(\"\\n========== END RAW ============================\")\n",
    "    lines = textwrap.wrap(data.pre[j].encode(\"utf-8\"), 50)\n",
    "    print(\"\\nPreprocessed:\\n\")\n",
    "    print(\"============ START PREPROCESSED =================\\n\")\n",
    "    for line in lines:\n",
    "        print(line)\n",
    "    print(\"========== END PREPROCESSED =====================\\n\")\n",
    "    print(\"\\nFeature vector (Top 15, sorted, non-zero values only):\")\n",
    "    print(\"Term, Index, Value:\")\n",
    "\n",
    "    Xa = tdm.toarray()[j]\n",
    "    sorted_ind = sorted(range(len(Xa)), key=lambda k: Xa[k])\n",
    "    for i, ind in enumerate(reversed(sorted_ind)):\n",
    "        if (i > 15): # Only print top terms\n",
    "            break\n",
    "        val = Xa[ind]\n",
    "        if val > 0:\n",
    "            print((\"{:>20s}, {:>5d}, {:>6.3f}\".format(ivocab[ind], ind, val)))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cluster 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document ID: 21\n",
      "Filename: 38760\n",
      "Truth category: 0 (comp.graphics)\n",
      "Assigned cluster: 0 (file format gif convert cview program ftp graphic pov)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: stusoft@hardy.u.washington.edu (Stuart Denman)\n",
      "Subject: Re: 3D2 files - what are they?\n",
      "Article-I.D.: shelley.1rft1nINNc7s\n",
      "Organization: University of Washington\n",
      "Lines: 16\n",
      "NNTP-Posting-Host: hardy.u.washington.edu\n",
      "\n",
      "doug@hparc0.aus.hp.com (Doug Parsons) writes:\n",
      "\n",
      ">I was chaining around in the anonymous ftp world looking for 3D Studio\n",
      ">meshes and other interesting graphical stuff for the program, and found\n",
      ">a few files with the extension 3D2.  My 3DS v2.01 doesn't know this type\n",
      ">of file, so what are they?\n",
      "\n",
      "They are 3D object files for CAD 3D 2.0, a program written by Tom Hudson\n",
      "for the Atari ST computers.  Don't know much more about them except that\n",
      "they are stored with the points first, then the surfaces are next, and are\n",
      "made by listing 3 point numbers that make up the triangle surface. Then\n",
      "there's a header that describes coloring, lighting, etc.  Don't know much\n",
      "more than this, hope this helps.\n",
      "\n",
      "Stuart Denman\n",
      "stusoft@u.washington.edu\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " 3d2 file articl shelley 1rft1ninnc7       chain\n",
      "around anonym ftp world look studio mesh interest\n",
      "graphic stuff program found file extens 3d2 3ds\n",
      "doesn't know type file  object file cad program\n",
      "written tom hudson atari comput don't know much\n",
      "except store point first surfac next made list\n",
      "point number make triangl surfac there header\n",
      "describ color light etc don't know much hope help\n",
      "stuart denman\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "                 3d2,    96,  0.423\n",
      "                file,  3477,  0.362\n",
      "              surfac,  8587,  0.229\n",
      "              stuart,  8464,  0.194\n",
      "             shelley,  7874,  0.183\n",
      "               atari,   735,  0.180\n",
      "                 3ds,    99,  0.170\n",
      "                know,  5052,  0.169\n",
      "                mesh,  5648,  0.168\n",
      "              hudson,  4423,  0.165\n",
      "             triangl,  9090,  0.163\n",
      "               chain,  1637,  0.163\n",
      "             program,  6905,  0.163\n",
      "              studio,  8470,  0.158\n",
      "                 cad,  1445,  0.155\n",
      "              header,  4217,  0.154\n",
      "Document ID: 47\n",
      "Filename: 38282\n",
      "Truth category: 0 (comp.graphics)\n",
      "Assigned cluster: 0 (file format gif convert cview program ftp graphic pov)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: rosa@ghost.dsi.unimi.it (massimo rossi)\n",
      "Subject: 3d studio works changes!!!!\n",
      "Organization: Computer Science Dep. - Milan University\n",
      "Lines: 28\n",
      "\n",
      " hi guys\n",
      " like all people in this group i'm a fans of fractal and render sw\n",
      " my favourite are fractint pov & 3dstudio 2.0 \n",
      " now listen my ideas\n",
      " i'have just starting now to be able to use 3dstudio quite well\n",
      " so i'm simulating a full animation of a f1 grand prix\n",
      " unfortanatly just some lap(10?)\n",
      " i' m very interested about all kind of .prj .3ds and so on\n",
      " concerning about cars or parts of its (motors wheel ...)\n",
      " (dxf are good enough)\n",
      " does anyone have object to give me to complete my hard animation\n",
      "\n",
      "\n",
      " anyway any exchanges about object material project will\n",
      " be VERY APRECIATE!!!!!\n",
      "\n",
      " is there a ftp site where I can find its?\n",
      "\n",
      " i' m looking for .pov files too\n",
      " (i 'm interested about cpu time comparision rendering images on\n",
      " pov & 3dstusio)\n",
      "\n",
      " thank to all\n",
      "\n",
      "\n",
      " email me at rosa@ghost.sm.dsi.unimi.it\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " studio work chang    guy like peopl group i'm fan\n",
      "fractal render favourit fractint pov 3dstudio\n",
      "listen idea i'hav start abl use 3dstudio quit well\n",
      "i'm simul full anim grand prix unfortanat lap\n",
      "interest kind prj 3ds concern car part motor wheel\n",
      "dxf good enough anyon object give complet hard\n",
      "anim   anyway exchang object materi project apreci\n",
      "ftp site find  look pov file interest cpu time\n",
      "comparis render imag pov 3dstusio  thank\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "                 pov,  6778,  0.399\n",
      "            3dstudio,   100,  0.386\n",
      "              render,  7277,  0.249\n",
      "                anim,   515,  0.214\n",
      "                 prj,  6882,  0.187\n",
      "            fractint,  3638,  0.183\n",
      "            comparis,  1961,  0.183\n",
      "              object,  6173,  0.182\n",
      "               wheel,  9721,  0.168\n",
      "                 3ds,    99,  0.160\n",
      "                 cpu,  2200,  0.154\n",
      "                 dxf,  2939,  0.153\n",
      "              studio,  8470,  0.150\n",
      "             exchang,  3264,  0.147\n",
      "             fractal,  3637,  0.145\n",
      "               grand,  3991,  0.142\n",
      "Document ID: 155\n",
      "Filename: 38326\n",
      "Truth category: 0 (comp.graphics)\n",
      "Assigned cluster: 0 (file format gif convert cview program ftp graphic pov)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: valo@cvtstu.cvt.stuba.cs (Valo Roman)\n",
      "Subject: Re: Text Recognition software availability\n",
      "Organization: Slovak Technical University Bratislava, Slovakia\n",
      "Lines: 23\n",
      "NNTP-Posting-Host: sk2eu.eunet.sk\n",
      "ReplyTo: valo@cvtstu.cvt.stuba.cs (Valo Roman)\n",
      "\n",
      "In article <C4IHqM.7v3@mentor.cc.purdue.edu>, ab@nova.cc.purdue.edu (Allen B) writes:\n",
      "|> One more time: is there any >free< OCR software out there?\n",
      "|>\n",
      "|> I ask this question periodically and haven't found anything.  This is\n",
      "|> the last time.  If I don't find anything, I'm going to write some\n",
      "|> myself.\n",
      "|> \n",
      "|> Post here or email me if you have any leads or suggestions, else just\n",
      "|> sit back and wait for me. :)\n",
      "|> \n",
      "|> ab\n",
      "\n",
      "I'm not sure if this is free or shareware, but you can try to look to wsmrsimtel20.army.mil,\n",
      "directory PD1:<MSDOS.DESKPUB>  file OCR104.ZIP .\n",
      "From the file SIMIBM.LST :\n",
      "OCR104.ZIP    B   93310  910424  Optical character recognition for scanners.\n",
      "\n",
      "Hope this helps.\n",
      "\n",
      "Roman Valo                                                          valo@cvt.stuba.cs\n",
      "Slovak Technical University\n",
      "Bratislava \n",
      "Slovakia\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " text recognit softwar avail       one time free\n",
      "ocr softwar  ask question period haven't found\n",
      "anyth last time don't find anyth i'm go write\n",
      "post email lead suggest els sit back wait    i'm\n",
      "sure free sharewar tri look wsmrsimtel20 armi mil\n",
      "directori pd1 msdos deskpub file ocr104 zip file\n",
      "simibm lst ocr104 zip optic charact recognit\n",
      "scanner  hope help   slovak technic univers\n",
      "bratislava slovakia\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "            recognit,  7175,  0.403\n",
      "                 zip,  9988,  0.322\n",
      "                free,  3668,  0.242\n",
      "                 pd1,  6491,  0.211\n",
      "                file,  3477,  0.206\n",
      "               anyth,   555,  0.204\n",
      "             softwar,  8103,  0.198\n",
      "                armi,   645,  0.177\n",
      "             scanner,  7663,  0.175\n",
      "                 mil,  5712,  0.171\n",
      "               optic,  6273,  0.168\n",
      "               msdos,  5903,  0.167\n",
      "            sharewar,  7856,  0.162\n",
      "             charact,  1661,  0.161\n",
      "                 sit,  7993,  0.151\n",
      "                wait,  9611,  0.143\n"
     ]
    }
   ],
   "source": [
    "idx_0 = [i for i, j in enumerate(km.labels_) if j == 0]\n",
    "printDoc(idx_0[0], data, km, cluster_terms)\n",
    "printDoc(idx_0[1], data, km, cluster_terms)\n",
    "printDoc(idx_0[2], data, km, cluster_terms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cluster 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document ID: 9\n",
      "Filename: 84309\n",
      "Truth category: 3 (talk.religion.misc)\n",
      "Assigned cluster: 1 (moral object valu say christian kuiper smiley subject peopl)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: markp@elvis.wri.com (Mark Pundurs)\n",
      "Subject: Re: After 2000 years, can we say that Christian Morality is\n",
      "Nntp-Posting-Host: elvis.wri.com\n",
      "Organization: Wolfram Research, Inc.\n",
      "Lines: 27\n",
      "\n",
      "In <930415.112243.8v6.rusnews.w165w@mantis.co.uk> mathew@mantis.co.uk (mathew) writes:\n",
      "\n",
      ">There's no objective physics; Einstein and Bohr have told us that.\n",
      "\n",
      "Speaking as one who knows relativity and quantum mechanics, I say: \n",
      "Bullshit.\n",
      "\n",
      ">There's no objective reality.  LSD should be sufficient to prove that.\n",
      "\n",
      "Speaking as one who has taken LSD, I say: \n",
      "Bullshit.\n",
      "\n",
      ">> One wonders just what people who ask such questions understand by the term \n",
      ">> \"objective\", if anything.\n",
      "\n",
      ">I consider it to be a useful fiction; an abstract ideal we can strive\n",
      ">towards.  Like an ideal gas or a light inextensible string, it doesn't\n",
      ">actually exist; but we can talk about things as if they were like it, and not\n",
      ">be too far wrong.\n",
      "\n",
      "How could striving toward an ideal be in any way useful, if the ideal \n",
      "had no objective existence?\n",
      "--\n",
      "Mark Pundurs\n",
      "\n",
      "any resemblance between my opinions and those \n",
      "of Wolfram Research, Inc. is purely coincidental\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " year say christian moral       there object\n",
      "physic einstein bohr told  speak one know relat\n",
      "quantum mechan say bullshit  there object realiti\n",
      "lsd suffici prove  speak one taken lsd say\n",
      "bullshit  one wonder peopl ask question understand\n",
      "term object anyth  consid use fiction abstract\n",
      "ideal strive toward like ideal gas light inextens\n",
      "string doesn't actual exist talk thing like far\n",
      "wrong  could strive toward ideal way use ideal\n",
      "object exist  mark pundur  resembl opinion wolfram\n",
      "research inc pure coincident\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "               ideal,  4483,  0.461\n",
      "              object,  6173,  0.299\n",
      "                 lsd,  5375,  0.288\n",
      "              strive,  8453,  0.279\n",
      "            bullshit,  1392,  0.268\n",
      "               speak,  8192,  0.179\n",
      "                 say,  7653,  0.154\n",
      "              pundur,  6985,  0.150\n",
      "             wolfram,  9816,  0.147\n",
      "                bohr,  1201,  0.147\n",
      "               exist,  3283,  0.145\n",
      "          coincident,  1884,  0.142\n",
      "            einstein,  3018,  0.125\n",
      "             quantum,  7025,  0.123\n",
      "              string,  8450,  0.121\n",
      "             fiction,  3463,  0.115\n",
      "Document ID: 41\n",
      "Filename: 83479\n",
      "Truth category: 3 (talk.religion.misc)\n",
      "Assigned cluster: 1 (moral object valu say christian kuiper smiley subject peopl)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: mccullou@snake2.cs.wisc.edu (Mark McCullough)\n",
      "Subject: Re: After 2000 years, can we say that Christian Morality is\n",
      "Organization: University of Wisconsin, Madison -- Computer Sciences Dept.\n",
      "Lines: 19\n",
      "\n",
      "In article <30136@ursa.bear.com> halat@pooh.bears (Jim Halat) writes:\n",
      ">Atoms are not objective.  They aren't even real.  What scientists call\n",
      ">an atom is nothing more than a mathematical model that describes \n",
      ">certain physical, observable properties of our surroundings.  All\n",
      ">of which is subjective.  \n",
      ">\n",
      ">-jim halat\n",
      "\n",
      "This deserves framing.  It really does.  \"[Atoms] aren't even real.\"\n",
      "\n",
      "Tell me then, those atoms we have seen with electron microscopes are\n",
      "atoms now, so what are they?  Figments of our imaginations?  The\n",
      "evidence that atoms are real is overwhelming, but I won't bother with\n",
      "most evidence at the moment.\n",
      "-- \n",
      "***************************************************************************\n",
      "* mccullou@whipple.cs.wisc.edu * Never program and drink beer at the same *\n",
      "* M^2                          *  time.  It doesn't work.                 *\n",
      "***************************************************************************\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " year say christian moral     atom object aren't\n",
      "even real scientist call atom noth mathemat model\n",
      "describ certain physic observ properti surround\n",
      "subject  jim halat  deserv frame realli atom\n",
      "aren't even real  tell atom seen electron\n",
      "microscop atom figment imagin evid atom real\n",
      "overwhelm won't bother evid moment    time doesn't\n",
      "work\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "                atom,   756,  0.723\n",
      "                real,  7144,  0.231\n",
      "                aren,   623,  0.196\n",
      "                evid,  3245,  0.190\n",
      "               halat,  4123,  0.166\n",
      "           microscop,  5690,  0.160\n",
      "           overwhelm,  6356,  0.149\n",
      "            surround,  8595,  0.126\n",
      "            properti,  6925,  0.123\n",
      "              deserv,  2560,  0.123\n",
      "             subject,  8487,  0.121\n",
      "              bother,  1245,  0.118\n",
      "            mathemat,  5545,  0.116\n",
      "           scientist,  7694,  0.114\n",
      "            electron,  3034,  0.111\n",
      "               frame,  3643,  0.109\n",
      "Document ID: 61\n",
      "Filename: 61067\n",
      "Truth category: 2 (sci.space)\n",
      "Assigned cluster: 1 (moral object valu say christian kuiper smiley subject peopl)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: chico@ccsun.unicamp.br (Francisco da Fonseca Rodrigues)\n",
      "Subject: New planet/Kuiper object found?\n",
      "X-Added: Forwarded by Space Digest\n",
      "Organization: [via International Space University]\n",
      "Original-Sender: isu@VACATION.VENARI.CS.CMU.EDU\n",
      "Distribution: sci\n",
      "Lines: 28\n",
      "\n",
      "\n",
      "\tTonigth a TV journal here in Brasil announced that an object,\n",
      "beyond Pluto's orbit, was found by an observatory at Hawaii. They\n",
      "named the object Karla.\n",
      "\n",
      "\tThe program said the object wasn't a gaseous giant planet, and\n",
      "should be composed by rocks and ices.\n",
      "\n",
      "\tCan someone confirm these information? Could this object be a\n",
      "new planet or a Kuiper object?\n",
      "\n",
      "\tThanks in advance.\n",
      "\n",
      "\tFrancisco.\n",
      "\n",
      "-----------------------=====================================----the stars,----\n",
      "|    ._,               | Francisco da Fonseca Rodrigues    |       o   o     |\n",
      "|  ,_| |._/\\           |                                   |     o         o |\n",
      "|  |       |o/^^~-._   | COTUCA-Colegio Tecnico da UNICAMP |   o             |\n",
      "|/-'    BRASIL      | ~|                                   |  o      o o     |\n",
      "|\\__/|_            /'  | Depto de Processamento de Dados   |  o    o  o  o   |\n",
      "|      \\__  Cps   | .  |                                   |   o  o  o    o  |\n",
      "|        |   * __/'    | InterNet : chico@ccsun.unicamp.br |     o o      o  |\n",
      "|        >   /'        |            cotuca@ccvax.unicamp.br|             o   |\n",
      "|      /'   /'         | Fone/Fax : 55-0192-32-9519        | o         o     |\n",
      "|     ~~^\\/'           | Campinas - SP - Brasil            |    o   o        |\n",
      "-----------------------=====================================----like dust.----\n",
      "\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " new planet kuiper object found ad forward space\n",
      "digest       tonigth journal brasil announc object\n",
      "beyond pluto orbit found observatori hawaii name\n",
      "object karla  program said object wasn't gaseous\n",
      "giant planet compos rock ice  someon confirm\n",
      "inform could object new planet kuiper object\n",
      "thank advanc  francisco  star francisco fonseca\n",
      "rodrigu  cotuca colegio tecnico unicamp brasil\n",
      "depto processamento dado cps   fone fax campina\n",
      "brasil like dust\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "              brasil,  1273,  0.497\n",
      "              object,  6173,  0.453\n",
      "              planet,  6664,  0.282\n",
      "              kuiper,  5080,  0.268\n",
      "           francisco,  3652,  0.232\n",
      "                 cps,  2199,  0.160\n",
      "             gaseous,  3782,  0.155\n",
      "                fone,  3581,  0.155\n",
      "               karla,  4958,  0.141\n",
      "              hawaii,  4208,  0.136\n",
      "              compos,  1981,  0.127\n",
      "                 ice,  4475,  0.124\n",
      "                dust,  2928,  0.121\n",
      "             confirm,  2032,  0.120\n",
      "                 new,  6023,  0.119\n",
      "             journal,  4898,  0.116\n"
     ]
    }
   ],
   "source": [
    "idx_1 = [i for i, j in enumerate(km.labels_) if j == 1]\n",
    "printDoc(idx_1[0], data, km, cluster_terms)\n",
    "printDoc(idx_1[1], data, km, cluster_terms)\n",
    "printDoc(idx_1[2], data, km, cluster_terms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cluster 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document ID: 0\n",
      "Filename: 102736\n",
      "Truth category: 1 (rec.sport.baseball)\n",
      "Assigned cluster: 2 (game team win pitch run score brave pitcher basebal)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: cubbie@garnet.berkeley.edu (                               )\n",
      "Subject: Re: Cubs behind Marlins? How?\n",
      "Article-I.D.: agate.1pt592$f9a\n",
      "Organization: University of California, Berkeley\n",
      "Lines: 12\n",
      "NNTP-Posting-Host: garnet.berkeley.edu\n",
      "\n",
      "\n",
      "gajarsky@pilot.njin.net writes:\n",
      "\n",
      "morgan and guzman will have era's 1 run higher than last year, and\n",
      " the cubs will be idiots and not pitch harkey as much as hibbard.\n",
      " castillo won't be good (i think he's a stud pitcher)\n",
      "\n",
      "       This season so far, Morgan and Guzman helped to lead the Cubs\n",
      "       at top in ERA, even better than THE rotation at Atlanta.\n",
      "       Cubs ERA at 0.056 while Braves at 0.059. We know it is early\n",
      "       in the season, we Cubs fans have learned how to enjoy the\n",
      "       short triumph while it is still there.\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " cub behind marlin articl agat 1pt592 f9a\n",
      "morgan guzman era run higher last year cub idiot\n",
      "pitch harkey much hibbard castillo won't good\n",
      "think he stud pitcher  season far morgan guzman\n",
      "help lead cub top era even better rotat atlanta\n",
      "cub era brave know earli season cub fan learn\n",
      "enjoy short triumph still\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "                 cub,  2282,  0.572\n",
      "                 era,  3164,  0.346\n",
      "              guzman,  4101,  0.306\n",
      "              morgan,  5861,  0.288\n",
      "                stud,  8467,  0.180\n",
      "              season,  7747,  0.178\n",
      "                agat,   327,  0.175\n",
      "            castillo,  1547,  0.171\n",
      "             triumph,  9108,  0.167\n",
      "              harkey,  4176,  0.167\n",
      "              marlin,  5517,  0.131\n",
      "               idiot,  4487,  0.128\n",
      "             atlanta,   749,  0.112\n",
      "               enjoy,  3114,  0.111\n",
      "               rotat,  7490,  0.107\n",
      "              higher,  4293,  0.106\n",
      "Document ID: 15\n",
      "Filename: 102617\n",
      "Truth category: 1 (rec.sport.baseball)\n",
      "Assigned cluster: 2 (game team win pitch run score brave pitcher basebal)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: kshus@schunix.uucp (Christopher Shustakg)\n",
      "Subject: Where can I find baseball statistics ??\n",
      "Organization: SCHUNIX Public Access Unix for Worcester County, MA, USA\n",
      "Lines: 7\n",
      "\n",
      "I am interested in uncovering statistics on Boston Red Sox players from\n",
      "March 1992 - present.  I want to look at changes in batting average, hits,\n",
      "multi-hit games, runs, stolen bases, and on base % during\n",
      "every game.  Where can\n",
      "I find this information?  Do any sports magazines log this info or do I\n",
      "have to go directly to the ball club?\n",
      "Thanks for the info.  Kip\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " find basebal statist    interest uncov statist\n",
      "boston red sox player march present want look\n",
      "chang bat averag hit multi hit game run stolen\n",
      "base base everi game find inform sport magazin log\n",
      "info direct ball club thank info kip\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "             statist,  8348,  0.361\n",
      "                info,  4636,  0.283\n",
      "                 hit,  4316,  0.251\n",
      "               uncov,  9233,  0.247\n",
      "                base,   923,  0.225\n",
      "                 log,  5328,  0.221\n",
      "                game,  3761,  0.218\n",
      "              stolen,  8407,  0.210\n",
      "               multi,  5917,  0.194\n",
      "               march,  5504,  0.193\n",
      "                club,  1842,  0.190\n",
      "             magazin,  5432,  0.185\n",
      "               sport,  8257,  0.174\n",
      "              boston,  1243,  0.173\n",
      "                 sox,  8162,  0.166\n",
      "              averag,   816,  0.153\n",
      "Document ID: 17\n",
      "Filename: 104921\n",
      "Truth category: 1 (rec.sport.baseball)\n",
      "Assigned cluster: 2 (game team win pitch run score brave pitcher basebal)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: st1rp@rosie.uh.edu (Schwam, David S.)\n",
      "Subject: Re: ASTROS FOR REAL?\n",
      "Organization: University of Houston\n",
      "Lines: 51\n",
      "Distribution: usa\n",
      "NNTP-Posting-Host: rosie.uh.edu\n",
      "News-Software: VAX/VMS VNEWS 1.41    \n",
      "\n",
      "In article <C5Ky58.12KD@austin.ibm.com>, marc@yogi.austin.ibm.com (Marc J. Stephenson) writes...\n",
      ">In article <1993Apr15.234838.4138@ccsvax.sfasu.edu> z_millerwl@ccsvax.sfasu.edu writes:\n",
      ">>WHO THINKS THE ASTROS ARE GOING PLACES???\n",
      ">>THEY'RE CURRENTLY FIRST PLACE.\n",
      ">>THEY'RE 5-4, 5-1 ON THE ROAD! \n",
      "> \n",
      ">I AGREE, LUMBERJACK (except that they're in 2nd)!  They ARE going PLACES -\n",
      ">San Diego, Los Angeles, San Francisco, Cincinnati, Denver, Atlanta, Miami,\n",
      ">Philadelphia, New York, Montreal, Pittsburgh, Chicago, St. Louis...and\n",
      ">points in between.  :-)\n",
      "> \n",
      ">But,\n",
      ">THEY'RE 0-3 AT HOME!\n",
      "\n",
      "But,\n",
      " THEY FACED THE PHILLIES -- A TEAM THAT GOT OFF TO AN 8-1 START.\n",
      "\n",
      "\n",
      "> \n",
      ">I'm just not used to an overly enthusiastic Houston fan.  I really shouldn't\n",
      ">discourage it, so HANG IN THERE, LUMBERJACK!  (But, get ahold of that shift\n",
      ">key, will ya?)\n",
      "> \n",
      ">ObBase:  Apparently the new owner (Drayton McLain (sp?)) doesn't particularly\n",
      ">like excuses.  An item in our paper (the Austin American-Statesman - \"If you\n",
      ">read it here, it was somewhere else first\") said that he wouldn't take \n",
      ">injuries as an excuse for losing because that possibility should have been\n",
      ">accounted for.  Uh, oh.  I don't want an owner that'll keep everybody on\n",
      ">edge - I'd never gotten that feeling about him, but who knows?  Does \n",
      "\n",
      "\n",
      "  To be honest, I think the city of Houston loves the new owner.  He has\n",
      "brought baseball back to Houston with key acquisitions -- players that\n",
      "were from the Houston area and wanted to play for the Astros.  I don't\n",
      "think that too many people are fearful that McLane will meddle in the team\n",
      "as he has already admitted that he doesn't know a whole lot about baseball.\n",
      "McLane is a businessman, and doesn't like excuses.  He makes a valid point\n",
      "that injuries shouldn't be an excuse to this club.  Look at the depth of the\n",
      "bench this season.. Canadele can play 7 positions; Bass and James are solid\n",
      "outfielders and can hit well too; Uribe is nice to have as well..  The\n",
      "pitching staff has 6 legitimate starters.  We're dealing with a young\n",
      "Houston team, so injuries shouldn't play a big role.  The only threat is\n",
      "the bullpen -- if Jonesy goes out, we may be in trouble but with the\n",
      "type of starters we have this season, there is less pressure on the pen.\n",
      "\n",
      "--- --- --- --- --- ---\n",
      " David S. Schwam\n",
      " University of Houston\n",
      " st1rp@jetson.uh.edu\n",
      "--- --- --- --- --- ---\n",
      "\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " astro real     news softwar vax vms vnew  yogi\n",
      "austin ibm com marc stephenson write  think astro\n",
      "go place they'r current first place they'r road\n",
      "agre lumberjack except they'r 2nd go place san\n",
      "diego los angel san francisco cincinnati denver\n",
      "atlanta miami philadelphia new york montreal\n",
      "pittsburgh chicago loui point   they'r home   face\n",
      "philli team got start    i'm use over enthusiast\n",
      "houston fan realli shouldn't discourag hang\n",
      "lumberjack get ahold shift key  obbas appar new\n",
      "owner drayton mclain doesn't particular like excus\n",
      "item paper austin american statesman read somewher\n",
      "els first said wouldn't take injuri excus lose\n",
      "possibl account don't want owner that'll keep\n",
      "everybodi edg i'd never gotten feel know   honest\n",
      "think citi houston love new owner brought basebal\n",
      "back houston key acquisit player houston area want\n",
      "play astro don't think mani peopl fear mclane\n",
      "meddl team alreadi admit doesn't know whole lot\n",
      "basebal mclane businessman doesn't like excus make\n",
      "valid point injuri shouldn't excus club look depth\n",
      "bench season canadel play posit bass jame solid\n",
      "outfield hit well urib nice well pitch staff\n",
      "legitim starter we'r deal young houston team\n",
      "injuri shouldn't play big role threat bullpen\n",
      "jonesi goe troubl type starter season less pressur\n",
      "pen   david schwam univers houston\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "             houston,  4399,  0.385\n",
      "               excus,  3268,  0.270\n",
      "               owner,  6359,  0.200\n",
      "              injuri,  4650,  0.195\n",
      "              mclane,  5583,  0.190\n",
      "          lumberjack,  5388,  0.190\n",
      "             shouldn,  7912,  0.190\n",
      "               astro,   724,  0.185\n",
      "              austin,   791,  0.160\n",
      "             starter,  8335,  0.126\n",
      "               place,  6657,  0.125\n",
      "                team,  8732,  0.122\n",
      "                play,  6681,  0.121\n",
      "               doesn,  2793,  0.119\n",
      "                 key,  4994,  0.112\n",
      "                 san,  7603,  0.110\n"
     ]
    }
   ],
   "source": [
    "idx_2 = [i for i, j in enumerate(km.labels_) if j == 2]\n",
    "printDoc(idx_2[0], data, km, cluster_terms)\n",
    "printDoc(idx_2[1], data, km, cluster_terms)\n",
    "printDoc(idx_2[2], data, km, cluster_terms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cluster 3\n",
    "This just looks like a general, unspecific cluster."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document ID: 3\n",
      "Filename: 38263\n",
      "Truth category: 0 (comp.graphics)\n",
      "Assigned cluster: 3 (like know think don time peopl new say right)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: davidr@rincon.ema.rockwell.com (David J. Ray)\n",
      "Subject: Re: Fractals? what good are they?\n",
      "Organization: Rockwell International\n",
      "X-Newsreader: Tin 1.1 PL5\n",
      "Lines: 16\n",
      "\n",
      "In regards to fractal commpression, I have seen 2 fractal compressed \"movies\".\n",
      "They were both fairly impressive.  The first one was a 64 gray scale \"movie\" of\n",
      "Casablanca, it was 1.3MB and had 11 minutes of 13 fps video.  It was a little\n",
      "grainy but not bad at all.  The second one I saw was only 3 minutes but it\n",
      "had 8 bit color with 10fps and measured in at 1.2MB.\n",
      "\n",
      "I consider the fractal movies a practical thing to explore.  But unlike many \n",
      "other formats out there, you do end up losing resolution.  I don't know what\n",
      "kind of software/hardware was used for creating the \"movies\" I saw but the guy\n",
      "that showed them to me said it took 5-15 minutes per frame to generate.  But as\n",
      "I said above playback was 10 or more frames per second.  And how else could you\n",
      "put 11 minutes on one floppy disk?\n",
      "\n",
      "davidr@rincon.ema.rockwell.com\n",
      "My opinions are my own except where they are shared by others in which case I \n",
      "will probably change my mind.\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " fractal good     regard fractal commpress seen\n",
      "fractal compress movi fair impress first one gray\n",
      "scale movi casablanca 3mb minut fps video littl\n",
      "graini bad second one saw minut bit color 10fps\n",
      "measur 2mb  consid fractal movi practic thing\n",
      "explor unlik mani format end lose resolut don't\n",
      "know kind softwar hardwar use creat movi saw guy\n",
      "show said took minut per frame generat said\n",
      "playback frame per second els could put minut one\n",
      "floppi disk   opinion except share other case\n",
      "probabl chang mind\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "             fractal,  3637,  0.475\n",
      "                movi,  5892,  0.414\n",
      "               minut,  5747,  0.375\n",
      "               frame,  3643,  0.196\n",
      "                 saw,  7650,  0.179\n",
      "                 2mb,    80,  0.154\n",
      "              second,  7753,  0.141\n",
      "                 fps,  3634,  0.140\n",
      "            playback,  6682,  0.136\n",
      "                said,  7583,  0.135\n",
      "              floppi,  3551,  0.126\n",
      "                gray,  4011,  0.115\n",
      "            compress,  1986,  0.105\n",
      "                disk,  2720,  0.104\n",
      "               share,  7855,  0.102\n",
      "               unlik,  9290,  0.101\n",
      "Document ID: 4\n",
      "Filename: 60824\n",
      "Truth category: 2 (sci.space)\n",
      "Assigned cluster: 3 (like know think don time peopl new say right)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: goltz@mimi.UU.NET (James P. Goltz)\n",
      "Subject: Orion drive in vacuum -- how?\n",
      "Organization: UUNET Technologies Inc, Falls Church, VA, USA\n",
      "Lines: 31\n",
      "NNTP-Posting-Host: mimi.uu.net\n",
      "\n",
      "\n",
      "  Background: The Orion spacedrive was a theoretical concept.  It\n",
      "would be a drive using thermonuclear explosions to drive a spacecraft.\n",
      "The idea was that you'd detonate devices with somewhere from one to\n",
      "ten megatons yield behind a \"pusher plate\" attached to the main\n",
      "spacecraft.  The shock wave from the explosions would transfer\n",
      "momentum to the ship.\n",
      "\n",
      "  Now, in an atmosphere I can see this.  The energy of the explosion\n",
      "heats the atmosphere, which expands explosively and slams a shock wave\n",
      "into the pusher plate.  But in a vacuum, only two things I can see are\n",
      "going to hit the plate: fission/fusion products (barium, krypton,\n",
      "helium, neutrons, evaporated bomb casing) and electromagnetic\n",
      "radiation (gammas mostly, some light/heat from irradiated fission\n",
      "products).\n",
      "\n",
      "  Would this work?  I can't see the EM radiation impelling very much\n",
      "momentum (especially given the mass of the pusher plate), and it seems\n",
      "to me you're going to get more momentum transfer throwing the bombs\n",
      "out the back of the ship than you get from detonating them once\n",
      "they're there.\n",
      "\n",
      "  I must be missing something.  Would someone enlighten me via email?\n",
      "\n",
      "  Thanks.\n",
      "\n",
      "-- \n",
      "\t--Jim\n",
      "\n",
      "---\n",
      "Jim Goltz                    AlterNet Engineer               goltz@uunet.uu.net\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " orion drive vacuum      background orion spacedr\n",
      "theoret concept drive use thermonuclear explos\n",
      "drive spacecraft idea you'd deton devic somewher\n",
      "one ten megaton yield behind pusher plate attach\n",
      "main spacecraft shock wave explos transfer\n",
      "momentum ship  atmospher see energi explos heat\n",
      "atmospher expand explos slam shock wave pusher\n",
      "plate vacuum two thing see go hit plate fission\n",
      "fusion product barium krypton helium neutron\n",
      "evapor bomb case electromagnet radiat gamma most\n",
      "light heat irradi fission product  work can't see\n",
      "radiat impel much momentum especi given mass\n",
      "pusher plate seem you'r go get momentum transfer\n",
      "throw bomb back ship get deton they'r  must miss\n",
      "someth someon enlighten via email  thank   jim\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "              pusher,  6997,  0.334\n",
      "              explos,  3308,  0.332\n",
      "            momentum,  5829,  0.299\n",
      "               plate,  6674,  0.290\n",
      "             fission,  3508,  0.211\n",
      "               deton,  2583,  0.207\n",
      "               shock,  7895,  0.187\n",
      "               drive,  2873,  0.186\n",
      "               orion,  6296,  0.185\n",
      "                bomb,  1209,  0.170\n",
      "              vacuum,  9405,  0.165\n",
      "                wave,  9661,  0.160\n",
      "                heat,  4226,  0.158\n",
      "            transfer,  9052,  0.154\n",
      "                ship,  7888,  0.154\n",
      "              radiat,  7058,  0.152\n",
      "Document ID: 5\n",
      "Filename: 83506\n",
      "Truth category: 3 (talk.religion.misc)\n",
      "Assigned cluster: 3 (like know think don time peopl new say right)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: Thyagi@cup.portal.com (Thyagi Morgoth NagaSiva)\n",
      "Subject: Re: OTO, the Ancient Order of Oriental Templars\n",
      "Organization: The Portal System (TM)\n",
      "Distribution: world\n",
      "  <1993Apr14.130150.28931@lynx.dac.northeastern.edu> <79615@cup.portal.com>\n",
      "  <1qn5rn$q7p@csugrad.cs.vt.edu>\n",
      "Lines: 144\n",
      "\n",
      "930418\n",
      "\n",
      "Do what thou wilt shall be the whole of the Law. [Honestly.]\n",
      "The word of Sin is Restriction. [Would I kid you?]\n",
      "\n",
      "\n",
      "Does one man's words encompass the majestic vision of thousands\n",
      "of individuals?  Quoting a man is not the same as quoting the\n",
      "Order.  Taken out of context, words can be interpreted much\n",
      "differently than had one applied them within the confines of\n",
      "their original expression.\n",
      "\n",
      "I think this is the case regarding Hymenaeus Beta, Frater Superior \n",
      "of the Order to which I belong.  When he included that bit\n",
      "from Merlinus X' he did us all a service.  He showed us the extremes\n",
      "to which Order members have been known to go in their fervor.\n",
      "I have little knowledge regarding Reuss' background, but surely\n",
      "he was an unusual man, and he was an important force in the Order \n",
      "for many years.\n",
      "\n",
      "Yet as people change so do Orders change, and while we look back\n",
      "so carefully at the dirty laundry of O.T.O. remember that this is\n",
      "only the surface skim and that many perspectives are now encompassed\n",
      "which extend beyond any one individual.  I hope to show that there\n",
      "was and is much room for a difference of opinion within the Order\n",
      "itself, perhaps by testing the limits myself.\n",
      "\n",
      "\n",
      "Let us examine this issue a bit more closely....\n",
      "\n",
      "\"In 1895, Karl Kellner (1850-1905), a wealthy Austrian industrialist\n",
      "and paper chemist, as well as a high-grade Mason, founded the Ordo\n",
      "Templi Orientis.  Kellner had traveled widely in the East, where he\n",
      "met three adepts who instructed him specific magical practices.  \n",
      "Kellner's efforts to develop the Order were later assisted by Franz\n",
      "Hartmann, Heinrich Klein and Theodore Reuss, who had worked together\n",
      "prior to joining the O.T.O.  The Order was first proclaimed in 1902\n",
      "in Reuss's Masonic publication, 'Oriflamme'.  On Kellner's death,\n",
      "Reuss succeeded him as Outer Head [O.H.O.].  The 'Jubilee' edition of\n",
      "the 'Oriflamme', published in 1912, announced that the Order taught\n",
      "secret of sexual magic.\n",
      " \n",
      "\"Theodore Reuss was an interesting character.  Born June 28, 1855 in\n",
      "Augsburg, he entered Masonry in 1876.  He was a singer, journalist and\n",
      "possibly a spy for the Prussian political police, infiltrating the Socialist\n",
      "League founded by Karl Marx's daughter and her husband.  Reuss was\n",
      "later associated with William Wynn Westcott, a leader of the Golden\n",
      "Dawn, who later introduced him to John Yarker.  Yarker chartered Reuss to\n",
      "found the Rites of Memphis and Mizraim in Germany.  After several\n",
      "attempts to concretize various Masonic Rites, Reuss settled on the\n",
      "development of the O.T.O.\n",
      "\n",
      "\"The Order experienced reasonably steady growth under Reuss' leadership.\n",
      "For example, he chartered Papus in France, Rudolph Steiner in Berlin\n",
      "and H. Spencer Lewis in the USA.  In 1912, the historic meeting between\n",
      "Reuss and Crowley occurred.  Crowley wrote that Reuss came to him and\n",
      "accused him of revealing Order secrets.  When Crowley looked at it afresh,\n",
      "the initiated interpretation of sexual magick unfolded itself to him for\n",
      "the first time.  Reuss appointed Crowley as Supreme and Holy King of all\n",
      "the English speaking world, and it was this authorization that he invoked\n",
      "when publishing the material of the Equinox.\n",
      "\n",
      "\"Reuss resigned as Outer Head of the Order in 1922 after suffering a\n",
      "stroke and named Crowley his successor.  All was well until 1925 when\n",
      "_The Book of the Law_ was translated into German.  There was a break\n",
      "in the continuity of the Order.  Manyk members split with the new O.H.O.\n",
      "over the book, which Crowley was actively promulgating through the Order.\n",
      "He had earlier revise dthe Order rituals at Reuss's request, deeply\n",
      "infusing the doctrines of the New Aeon revelation.\"\n",
      "\n",
      "_An Introduction to the History of the O.T.O._, by Ad Veritatem IX'\n",
      "\n",
      "Within _Equinox III:10_, Edited by \n",
      "  Hymenaeus Beta, Frater Superior, Rex Summus Sanctissimus,\n",
      "    Caliph of the United States of America,\n",
      "      Published by Samuel Weiser, 1990.\n",
      "\n",
      "\n",
      "\n",
      "There are many possible reasons that our Frater Superior included this\n",
      "material in _Equinox III:10_.  And this is the real point, is it not?\n",
      "Why did he wish to publish such things about the history of his own\n",
      "organization?  Does he represent a dogmatic threat to the principle\n",
      "of Thelema?  Or is he exercising his True Will and putting forth very\n",
      "complex pictures with no easy answers?  A picture which leaves room\n",
      "for very many interpretations.\n",
      "\n",
      "It is quite easy for me to see, for example, that all of O.T.O. derived\n",
      "out of the dribble of faltering Masonry, purchased by clever hucksters\n",
      "with an ounce of courage and some writing ability to aid them.  And I\n",
      "can take that all the way down to our present Caliph, whose feeble\n",
      "support of the 'Law of Thelema' is laughable at best.\n",
      "\n",
      "Would I be thrown out of the Order for speaking in this way?  \n",
      "Will I?  \n",
      "I think not.\n",
      "Why?  Because my Frater will see it as a perspective, an interjection\n",
      "I am using as an example.  My illustration shows that we may express things\n",
      "in the context of a larger work and the true significance of this may be\n",
      "quite difficult to apprehend at first.\n",
      "\n",
      "So it may be with OTO and Merlinus X'.  Please look O.T.O. more carefully.\n",
      "I do not support Reuss's words myself, as I am not qualified to assess\n",
      "them, and I am critical of their pomposity.  If I who am a member of\n",
      "the Order take such a stand and am allowed to continue doing so, then\n",
      "what can this say about the health of the Order?  Does it mean that\n",
      "the Order has 'gone soft' and abandoned its moral principles?  Or\n",
      "does it mean that it is strong in its ability to let the will of\n",
      "universal kinship arise on its own, not shackled by some dogmatic\n",
      "requirement?  How shall we resolve these two possibilities?\n",
      "\n",
      "\n",
      "I find a high calibre of individual associated with Ordo Templi Orientis.\n",
      "They are often quite intelligent and sometimes very well versed in arcane\n",
      "or usual information.  They are quite often artists and geniuses.  \n",
      "Having met some 20 longstanding members in the SF Bay Area (many who are or\n",
      "were very heavily involved with the Order), I can vouch for the integrity\n",
      "of the organization as it stands.\n",
      "\n",
      "I have sometimes questioned the policy of Hymenaeus Beta.  In these moments \n",
      "I followed my intuition, and I've found little to stop me from requesting\n",
      "a Second initiation from a different O.T.O. body.  I'm happily participating\n",
      "in social groups (Feasts or Initiations) and have come to know the Gnostic\n",
      "Mass well enough for my tastes.\n",
      " \n",
      "\n",
      "This doesn't make me an authority on Order politics and explanations, however.\n",
      "I can only hypothesize and relay to you what I understand based on my\n",
      "limited contact with other members.\n",
      "\n",
      "I urge you not to take the words of Merlinus X too far.  There are many\n",
      "ways to interpret words, and many people who have become involved with\n",
      "the Order feel very strongly about the sanctity of personal freedom\n",
      "and the preservation of individual vision.\n",
      "\n",
      "I welcome other comment on this issue and will be writing more in response\n",
      "to other posts in this thread.\n",
      "\n",
      "\n",
      "Invoke me under my stars.  Love is the law, love under will.\n",
      "\n",
      "I am I!\n",
      "\n",
      "Frater (I) Nigris (DCLXVI) CCCXXXIII\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " oto ancient order orient templar   cup portal com\n",
      "thou wilt shall whole law honest word sin restrict\n",
      "kid   one man word encompass majest vision\n",
      "thousand individu quot man quot order taken\n",
      "context word interpret much differ one appli\n",
      "within confin origin express  think case regard\n",
      "hymenaeus beta frater superior order belong includ\n",
      "bit merlinus servic show extrem order member known\n",
      "fervor littl knowledg regard reuss background sure\n",
      "unusu man import forc order mani year  yet peopl\n",
      "chang order chang look back care dirti laundri\n",
      "rememb surfac skim mani perspect encompass extend\n",
      "beyond one individu hope show much room differ\n",
      "opinion within order perhap test limit   let\n",
      "examin issu bit close  karl kellner wealthi\n",
      "austrian industrialist paper chemist well high\n",
      "grade mason found ordo templi orienti kellner\n",
      "travel wide east met three adept instruct specif\n",
      "magic practic kellner effort develop order later\n",
      "assist franz hartmann heinrich klein theodor reuss\n",
      "work togeth prior join order first proclaim reuss\n",
      "mason public oriflamm kellner death reuss succeed\n",
      "outer head jubile edit oriflamm publish announc\n",
      "order taught secret sexual magic  theodor reuss\n",
      "interest charact born augsburg enter masonri\n",
      "singer journalist possibl spi prussian polit polic\n",
      "infiltr socialist leagu found karl marx daughter\n",
      "husband reuss later associ william wynn westcott\n",
      "leader golden dawn later introduc john yarker\n",
      "yarker charter reuss found rite memphi mizraim\n",
      "germani sever attempt concret various mason rite\n",
      "reuss settl develop  order experienc reason steadi\n",
      "growth reuss leadership exampl charter papus franc\n",
      "rudolph steiner berlin spencer lewi usa histor\n",
      "meet reuss crowley occur crowley wrote reuss came\n",
      "accus reveal order secret crowley look afresh\n",
      "initi interpret sexual magick unfold first time\n",
      "reuss appoint crowley suprem holi king english\n",
      "speak world author invok publish materi equinox\n",
      "reuss resign outer head order suffer stroke name\n",
      "crowley successor well book law translat german\n",
      "break continu order manyk member split new book\n",
      "crowley activ promulg order earlier revis dthe\n",
      "order ritual reuss request deepli infus doctrin\n",
      "new aeon revel  introduct histori veritatem ix\n",
      "within equinox iii edit hymenaeus beta frater\n",
      "superior rex summus sanctissimus caliph unit state\n",
      "america publish samuel weiser    mani possibl\n",
      "reason frater superior includ materi equinox iii\n",
      "real point wish publish thing histori organ repres\n",
      "dogmat threat principl thelema exercis true put\n",
      "forth complex pictur easi answer pictur leav room\n",
      "mani interpret  quit easi see exampl deriv dribbl\n",
      "falter masonri purchas clever huckster ounc courag\n",
      "write abil aid take way present caliph whose feebl\n",
      "support law thelema laughabl best  thrown order\n",
      "speak way  think frater see perspect interject use\n",
      "exampl illustr show express thing context larger\n",
      "work true signific quit difficult apprehend first\n",
      "oto merlinus pleas look care support reuss word\n",
      "qualifi assess critic pompos member order take\n",
      "stand allow continu say health order mean order\n",
      "gone soft abandon moral principl mean strong abil\n",
      "let univers kinship aris shackl dogmat requir\n",
      "shall resolv two possibl   find high calibr\n",
      "individu associ ordo templi orienti often quit\n",
      "intellig sometim well vers arcan usual inform quit\n",
      "often artist genius met longstand member bay area\n",
      "mani heavili involv order vouch integr organ stand\n",
      "sometim question polici hymenaeus beta moment\n",
      "follow intuit i'v found littl stop request second\n",
      "initi differ bodi i'm happili particip social\n",
      "group feast initi come know gnostic mass well\n",
      "enough tast   doesn't make author order polit\n",
      "explan howev hypothes relay understand base limit\n",
      "contact member  urg take word merlinus far mani\n",
      "way interpret word mani peopl becom involv order\n",
      "feel strong sanctiti person freedom preserv\n",
      "individu vision  welcom comment issu write respons\n",
      "post thread   invok star love law love    frater\n",
      "nigri dclxvi cccxxxiii\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "               reuss,  7350,  0.546\n",
      "               order,  6281,  0.400\n",
      "             crowley,  2246,  0.224\n",
      "              frater,  3661,  0.192\n",
      "             equinox,  3160,  0.109\n",
      "              member,  5625,  0.109\n",
      "                word,  9829,  0.106\n",
      "               mason,  5531,  0.098\n",
      "                mani,  5479,  0.098\n",
      "           interpret,  4720,  0.093\n",
      "             publish,  6974,  0.091\n",
      "                beta,  1055,  0.089\n",
      "            superior,  8562,  0.088\n",
      "            individu,  4604,  0.086\n",
      "                 law,  5153,  0.085\n",
      "             theodor,  8843,  0.080\n"
     ]
    }
   ],
   "source": [
    "idx_3 = [i for i, j in enumerate(km.labels_) if j == 3]\n",
    "printDoc(idx_3[0], data, km, cluster_terms)\n",
    "printDoc(idx_3[1], data, km, cluster_terms)\n",
    "printDoc(idx_3[2], data, km, cluster_terms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cluster 7\n",
    "This looks like something to do with Jewish baseball players."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document ID: 22\n",
      "Filename: 104933\n",
      "Truth category: 1 (rec.sport.baseball)\n",
      "Assigned cluster: 7 (jewish player basebal lowenstein come hank koufax stankowitz sandi)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: roger@crux.Princeton.EDU (Roger Lustig)\n",
      "Subject: Re: Jewish Baseball Players?\n",
      "Originator: news@nimaster\n",
      "Nntp-Posting-Host: crux.princeton.edu\n",
      "Reply-To: roger@astro.princeton.edu (Roger Lustig)\n",
      "Organization: Princeton University\n",
      "Lines: 24\n",
      "\n",
      "In article <15APR93.14691229.0062@lafibm.lafayette.edu> VB30@lafibm.lafayette.edu (VB30) writes:\n",
      ">Just wondering.  A friend and I were talking the other day, and\n",
      ">we were (for some reason) trying to come up with names of Jewish\n",
      ">baseball players, past and present.  We weren't able to come up\n",
      ">with much, except for Sandy Koufax, (somebody) Stankowitz, and\n",
      ">maybe John Lowenstein.  Can anyone come up with any more.  I know\n",
      ">it sounds pretty lame to be racking our brains over this, but\n",
      ">humor us.  Thanks for your help.\n",
      "\n",
      "Stankiewicz?  I doubt it.\n",
      "\n",
      "Koufax was one of two Jewish HOFs: the other is Hank Greenberg.\n",
      "\n",
      "Other good players: Buddy Myer, Johnny Kling, Norm and Larry Sherry,\n",
      "Ken Holtzman, Saul Rogovin, Ed Reulbach.  \n",
      "\n",
      "There have been over 150 Jewish major leaguers.  A few years ago there\n",
      "was an article about someone who keeps track of this in Spy magazine;\n",
      "the article was entitled \"Jews on First,\" of course.\n",
      "\n",
      "There have also been at least two books on the subject.\n",
      "\n",
      "Roger\n",
      "\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " jewish basebal player origin news nimast\n",
      "wonder friend talk day reason tri come name jewish\n",
      "basebal player past present weren't abl come much\n",
      "except sandi koufax somebodi stankowitz mayb john\n",
      "lowenstein anyon come know sound pretti lame rack\n",
      "brain humor thank help  stankiewicz doubt  koufax\n",
      "one two jewish hof hank greenberg  good player\n",
      "buddi myer johnni kling norm larri sherri ken\n",
      "holtzman saul rogovin reulbach  jewish major\n",
      "leaguer year ago articl someon keep track spi\n",
      "magazin articl entitl jew first cours  also least\n",
      "two book  roger\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "              jewish,  4864,  0.404\n",
      "              koufax,  5067,  0.262\n",
      "              player,  6683,  0.213\n",
      "                come,  1923,  0.179\n",
      "            holtzman,  4353,  0.161\n",
      "            reulbach,  7347,  0.156\n",
      "         stankiewicz,  8320,  0.152\n",
      "                saul,  7645,  0.149\n",
      "                norm,  6107,  0.149\n",
      "                myer,  5939,  0.146\n",
      "                 hof,  4336,  0.144\n",
      "             leaguer,  5177,  0.142\n",
      "               buddi,  1366,  0.142\n",
      "                 spi,  8229,  0.140\n",
      "             basebal,   924,  0.139\n",
      "              johnni,  4879,  0.136\n",
      "Document ID: 26\n",
      "Filename: 104911\n",
      "Truth category: 1 (rec.sport.baseball)\n",
      "Assigned cluster: 7 (jewish player basebal lowenstein come hank koufax stankowitz sandi)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: Eastgate@world.std.com (Mark Bernstein)\n",
      "Subject: Jewish Broadcasters (was Jewish Baseball Players?)\n",
      "Organization: The World Public Access UNIX, Brookline, MA\n",
      "Lines: 8\n",
      "\n",
      "For that matter, how many Gentleman of The Press Box have been Jewish? The\n",
      "only Jewish sportscaster that comes to mind is Steve Williams (?), who had\n",
      "a Phillies show on KYW in Philadelphia in the 80s.\n",
      "-- \n",
      "Mark Bernstein\n",
      "Eastgate Systems, Inc.   134 Main Street   Watertown MA 02172 USA\n",
      "voice: (800) 562-1638 in USA   +1(617) 924-9044\n",
      "Eastgate@world.std.com    Compuserve: 76146,262    AppleLink:Eastgate \n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " jewish broadcast jewish basebal player    matter\n",
      "mani gentleman press box jewish jewish sportscast\n",
      "come mind steve william philli show kyw\n",
      "philadelphia 80s  mark bernstein eastgat system\n",
      "inc main street watertown usa voic usa\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "              jewish,  4864,  0.613\n",
      "                 usa,  9357,  0.300\n",
      "           bernstein,  1046,  0.244\n",
      "          sportscast,  8258,  0.237\n",
      "           gentleman,  3832,  0.231\n",
      "                 80s,   144,  0.231\n",
      "           broadcast,  1324,  0.163\n",
      "        philadelphia,  6589,  0.162\n",
      "              street,  8440,  0.160\n",
      "                voic,  9567,  0.158\n",
      "             william,  9764,  0.157\n",
      "               press,  6841,  0.150\n",
      "              philli,  6593,  0.150\n",
      "               steve,  8390,  0.139\n",
      "                main,  5450,  0.137\n",
      "                 box,  1254,  0.128\n",
      "Document ID: 34\n",
      "Filename: 104391\n",
      "Truth category: 1 (rec.sport.baseball)\n",
      "Assigned cluster: 7 (jewish player basebal lowenstein come hank koufax stankowitz sandi)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: roger@crux.Princeton.EDU (Roger Lustig)\n",
      "Subject: Re: Jewish Baseball Players?\n",
      "Originator: news@nimaster\n",
      "Nntp-Posting-Host: crux.princeton.edu\n",
      "Reply-To: roger@astro.princeton.edu (Roger Lustig)\n",
      "Organization: Princeton University\n",
      "Lines: 16\n",
      "\n",
      "In article <1993Apr16.200100.23703@scott.skidmore.edu> jrogoff@scott.skidmore.edu (jay rogoff) writes:\n",
      "\n",
      ">There is a book that you can find in sale catalogues occasionally\n",
      ">called _Jewish Baseball Stars_, and baseball mavin Peter Bjarkman has\n",
      ">compiled a pretty definitive list of Jewish ballplayers in the bigs.  \n",
      "\n",
      "I wish I hadn't sold my copy of Jewish Baseball Stars.  It's a Short Shelf\n",
      "(i.e., the one on top of the toilet tank) Special.  The writing in that \n",
      "books is so astonishingly awful -- every sportswriting cliche taken to\n",
      "the nth degree and then mangled -- that it's funny.\n",
      "\n",
      "Rusinow is the author, I think.\n",
      "\n",
      "Roger\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " jewish basebal player origin news nimast\n",
      "book find sale catalogu occasion call jewish\n",
      "basebal star basebal mavin peter bjarkman compil\n",
      "pretti definit list jewish ballplay big  wish\n",
      "hadn't sold copi jewish basebal star it short\n",
      "shelf one top toilet tank special write book\n",
      "astonish aw everi sportswrit clich taken nth degre\n",
      "mangl it funni  rusinow author think  roger\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "              jewish,  4864,  0.470\n",
      "             basebal,   924,  0.323\n",
      "                star,  8324,  0.207\n",
      "               mangl,  5477,  0.194\n",
      "               clich,  1814,  0.181\n",
      "            astonish,   717,  0.181\n",
      "              toilet,  8970,  0.177\n",
      "                 nth,  6145,  0.173\n",
      "                book,  1218,  0.171\n",
      "          sportswrit,  8259,  0.170\n",
      "            catalogu,  1553,  0.165\n",
      "               shelf,  7872,  0.152\n",
      "            ballplay,   884,  0.147\n",
      "                hadn,  4114,  0.143\n",
      "            occasion,  6187,  0.142\n",
      "              nimast,  6072,  0.141\n"
     ]
    }
   ],
   "source": [
    "idx_7 = [i for i, j in enumerate(km.labels_) if j == 7]\n",
    "printDoc(idx_7[0], data, km, cluster_terms)\n",
    "printDoc(idx_7[1], data, km, cluster_terms)\n",
    "printDoc(idx_7[2], data, km, cluster_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document ID: 999\n",
      "Filename: 61027\n",
      "Truth category: 2 (sci.space)\n",
      "Assigned cluster: 4 (graphic thank mail pleas look use program polygon help)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: wallacen@CS.ColoState.EDU (nathan wallace)\n",
      "Subject:  Level 5\n",
      "Reply-To: wallacen@CS.ColoState.EDU\n",
      "Nntp-Posting-Host: sor.cs.colostate.edu\n",
      "Organization: Colorado State University -=- Computer Science Dept.\n",
      "Lines: 15\n",
      "\n",
      "According to a Software engineering professor here, what was actually rated\n",
      "level five was an ibm unit which produced part of the software for the shuttle,\n",
      "by not means all of it. \n",
      "\n",
      "Interesting note: 90% of the software development groups surveyed were at\n",
      "level 1. The ibm shuttle groups was the *only* one at level 5!\n",
      "\n",
      "---\n",
      "C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/\n",
      "C/    Nathan F. Wallace                   C/C/           \"Reality Is\"         C/\n",
      "C/    e-mail: wallacen@cs.colostate.edu   C/C/    ancient Alphaean proverb    C/\n",
      "C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/C/\n",
      " \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " level      accord softwar engin professor actual\n",
      "rate level five ibm unit produc part softwar\n",
      "shuttl mean  interest note softwar develop group\n",
      "survey level ibm shuttl group one level    nathan\n",
      "wallac realiti\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "               level,  5229,  0.560\n",
      "             softwar,  8103,  0.342\n",
      "                 ibm,  4472,  0.330\n",
      "              shuttl,  7929,  0.275\n",
      "              wallac,  9620,  0.252\n",
      "               group,  4064,  0.237\n",
      "              survey,  8597,  0.195\n",
      "           professor,  6901,  0.194\n",
      "             realiti,  7148,  0.160\n",
      "                unit,  9278,  0.155\n",
      "              accord,   219,  0.153\n",
      "              produc,  6895,  0.151\n",
      "                rate,  7108,  0.148\n",
      "               engin,  3109,  0.133\n",
      "             develop,  2588,  0.127\n",
      "                note,  6118,  0.123\n"
     ]
    }
   ],
   "source": [
    "printDoc(999, data, km, cluster_terms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Look at Some Mismatches/Errors\n",
    "\n",
    "Since we conveniently have the truth categories that each document came from, we can (sort of) look at some documents that were not assigned to an expected cluster, by looking at the confusion matrix above. For example, if a given category had 99% of its documents assigned to cluster 3, then we can look at the 1% that weren't assigned to cluster three to see what the problem is. Likewise, if a given cluster contains documents mostly from category 1, then we can look at some of the documents in the cluster that didn't come from category 1.\n",
    "\n",
    "\n",
    "First, we'll define a quick function to do the work."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def findMismatches(target_id, label_id, data):\n",
    "    '''\n",
    "    Given a truth category (i.e., target_id) and an assigned clusters id (i.e., label_id), find\n",
    "    document_ids that match the label_id but *not* the target_id. \n",
    "    '''\n",
    "    mismatches = []\n",
    "    for i in range(0, len(data.pre)):\n",
    "        if (data.target[i] != target_id and data.assigned_cluster[i] == label_id):\n",
    "            mismatches.append(i)\n",
    "    return mismatches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cluster 0/Category 0\n",
    "\n",
    "Looking at the confusion matrix above, it looks like most documents assigned to cluster 0 (\"file format gif convert cview program\") come from category 0 (comp.graphics). Let's look at documents assigned to cluster 0 that aren't from category 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Document ID: 2519\n",
      "Filename: 104886\n",
      "Truth category: 1 (rec.sport.baseball)\n",
      "Assigned cluster: 0 (file format gif convert cview program ftp graphic pov)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: cruz@stsci.edu\n",
      "Subject: Re: Baseball logos available via anonymous FTP\n",
      "Lines: 31\n",
      "Organization: Space Telescope Science Institute\n",
      "Distribution: world,na\n",
      "\n",
      "In article <1993Apr21.165206.13060@ac.dal.ca>, arishem@ac.dal.ca writes:\n",
      "> \n",
      "> Well, thanks to Matthew Wall, the Major League baseball logos which I've been\n",
      "> posting over the past number of weeks now have a home at the Internet\n",
      "> Baseball Archive.\n",
      "> To get them, FTP to eucalyptus.cc.swarthmore.edu, login anonymously as per\n",
      "> usual, and cd to the baseball/misc/graphics directory.  Both the GIF files\n",
      "> and the uuencoded versions are available.\n",
      "> \n",
      ">   Darren\n",
      "> \n",
      ">  Darren Reiniger                   reiniger@ug.cs.dal.ca || arishem@ac.dal.ca\n",
      ">  Centre For Marine Geology, Dalhousie University, Halifax, N.S., Canada\n",
      "> | People who wonder where this generation is going should remind themselves   |\n",
      "> | where it came from in the first place.                                      |\n",
      "\n",
      "\n",
      "I have successfully copied all of the uuencoded versions of these logos to my\n",
      "local directory, but when I tried the GIF files, it said \"Permission denied\".\n",
      "Could you explain why, and also how to execute the uuencoded files?\n",
      "\n",
      "Thanks,\n",
      "\n",
      "Ken\n",
      "\n",
      "\n",
      "================================================================================\n",
      "% KEN CRUZ                   | 1993 NFL CHAMPS: DALLAS COWBOYS                 %\n",
      "% JOHNS HOPKINS UNIVERSITY   | 1993 MLB CHAMPS: BALTIMORE ORIOLES              %\n",
      "% BALTIMORE, MARYLAND        | 1993 ALL-STAR GAME @ ORIOLE PARK at CAMDEN YARDS%\n",
      "================================================================================\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " basebal logo avail via anonym ftp       well\n",
      "thank matthew wall major leagu basebal logo i'v\n",
      "post past number week home internet basebal archiv\n",
      "get ftp eucalyptus swarthmor edu login anonym per\n",
      "usual basebal misc graphic directori gif file\n",
      "uuencod version avail  darren  dal centr marin\n",
      "geolog dalhousi univers halifax canada peopl\n",
      "wonder generat go remind came first place\n",
      "success copi uuencod version logo local directori\n",
      "tri gif file said permiss deni could explain also\n",
      "execut uuencod file  thank  ken    ken cruz nfl\n",
      "champ dalla cowboy john hopkin univers mlb champ\n",
      "baltimor oriol baltimor maryland star game oriol\n",
      "park camden yard\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "             uuencod,  9389,  0.344\n",
      "                logo,  5332,  0.328\n",
      "               champ,  1645,  0.233\n",
      "             basebal,   924,  0.228\n",
      "               oriol,  6295,  0.181\n",
      "                file,  3477,  0.175\n",
      "            baltimor,   887,  0.171\n",
      "              anonym,   532,  0.168\n",
      "                 ken,  4980,  0.166\n",
      "           directori,  2669,  0.160\n",
      "                 gif,  3869,  0.143\n",
      "                 ftp,  3709,  0.137\n",
      "           swarthmor,  8622,  0.132\n",
      "             halifax,  4127,  0.132\n",
      "                cruz,  2263,  0.132\n",
      "             version,  9479,  0.131\n",
      "\n",
      "Document ID: 2723\n",
      "Filename: 104960\n",
      "Truth category: 1 (rec.sport.baseball)\n",
      "Assigned cluster: 0 (file format gif convert cview program ftp graphic pov)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: wall@cc.swarthmore.edu (Matthew Wall)\n",
      "Subject: Re: mlb.c\n",
      "Nntp-Posting-Host: mac8.beardsley.swarthmore.edu\n",
      "Organization: Hi, I'm Matt, and I'm a...a...Boston Red Sox fan\n",
      "Distribution: usa\n",
      "Lines: 30\n",
      "\n",
      "In article <5086@cvbnetPrime.COM>, cvalcour@ptex.Prime.COM (Sorry I missed\n",
      "the posting) wrote:\n",
      "> \n",
      "> Could some kind soul please email me a copy of mlb.c\n",
      "> \n",
      "> Thanks\n",
      "> Chris V.\n",
      "> ------------------------------------------------------\n",
      "> cvalcour@ptex.Prime.COM\n",
      "> ------------------------------------------------------\n",
      "\n",
      "\n",
      "mlb.c is archived on the Internet Baseball Archive\n",
      "\n",
      "anoymous ftp to eucalyptus.cc.swarthmore.edu (130.58.67.44)\n",
      "you must use your full email / username id as a password\n",
      "\n",
      "cd ./baseball/schedules\n",
      "\n",
      "it's filed as mlb-schedule-program.c for clarity\n",
      "\n",
      "schedules pre-generated for east-coast times are there as text files as\n",
      "well, and a couple of teams' TV schedules -- please send me more TV info if\n",
      "you have it on other teams.\n",
      "\n",
      "- matt\n",
      "\n",
      "Matt Wall * wall@cc.swarthmore.edu * Hey, I gotta job here, OK?\n",
      "---------------------------------------------------------------\n",
      "April 6, 1993: Boston Red Sox seize first place.\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " mlb      ptex prime com sorri miss post wrote\n",
      "could kind soul pleas email copi mlb  thank chris\n",
      "mlb archiv internet basebal archiv  anoym ftp\n",
      "eucalyptus swarthmor edu must use full email\n",
      "usernam password  basebal schedul  it file mlb\n",
      "schedul program clariti  schedul pre generat east\n",
      "coast time text file well coupl team schedul pleas\n",
      "send info team  matt    april boston red sox seiz\n",
      "first place\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "                 mlb,  5795,  0.498\n",
      "             schedul,  7678,  0.416\n",
      "              archiv,   618,  0.225\n",
      "               email,  3060,  0.192\n",
      "           swarthmor,  8622,  0.178\n",
      "                seiz,  7772,  0.165\n",
      "                file,  3477,  0.158\n",
      "             usernam,  9368,  0.155\n",
      "             basebal,   924,  0.154\n",
      "                team,  8732,  0.152\n",
      "            password,  6455,  0.141\n",
      "               pleas,  6688,  0.134\n",
      "               coast,  1859,  0.133\n",
      "               prime,  6865,  0.130\n",
      "                matt,  5552,  0.127\n",
      "                soul,  8149,  0.126\n"
     ]
    }
   ],
   "source": [
    "m = findMismatches(0, 0, data)\n",
    "m\n",
    "print(\"\")\n",
    "printDoc(m[0], data, km, cluster_terms)\n",
    "print(\"\")\n",
    "printDoc(m[1], data, km, cluster_terms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cluster 9/Category 2\n",
    "\n",
    "Now, let's look at the single document that was assigned to cluster 9 (space moon launch nasa cost station shuttl billion sky like) and didn't come from category 2 (sci.space)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Document ID: 2801\n",
      "Filename: 39496\n",
      "Truth category: 0 (comp.graphics)\n",
      "Assigned cluster: 9 (space moon launch nasa cost station shuttl billion sky)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: wnkretz@ikesg1.energietechnik.uni-stuttgart.de (Oliver Kretzschmar)\n",
      "Subject: Need C-algorithm, which decide whether a 3D-point is inside a space, defined by 6 planes\n",
      "Organization: IKE\n",
      "Lines: 31\n",
      "Distribution: world\n",
      "NNTP-Posting-Host: ikesg1.energietechnik.uni-stuttgart.de\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "   Hey,\n",
      "\n",
      "   I am looking for C-algorithm, which decide,whether a 3D-Point is\n",
      "   inside a space which was defined by 8 x 3D-points.\n",
      "   With a space defined by 8 x 3D-points, I mean a space defined by\n",
      "   6 closed 3D meshes defined by 8 points. Like this :\n",
      "\n",
      "              _____\n",
      "             /    /|\n",
      "            /____/ |\n",
      "            |    | |\n",
      "            |    | /\n",
      "            |____|/\n",
      "\n",
      "\n",
      "   But in this simple example, the sides are parallel, but my problem,\n",
      "   the sides can be non parallel and the opposite sides must not have\n",
      "   the same size etc. Please mail me your informations !\n",
      "\n",
      "   Thanks for your great efforts\n",
      "\n",
      "      Oliver\n",
      "\n",
      "-- \n",
      " NAME : O.Kretzschmar                      Inst.IKE / University Stuttgart\n",
      " PHONE: +49 711 685 2130                           Pfaffenwaldring 31\n",
      " FAX  : +49 711 685 2010                            7000 Stuttgart 80\n",
      " EMAIL: wnkretz@ikesg1.energietechnik.uni-stuttgart.de\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " need algorithm decid whether point insid space\n",
      "defin plane         hey  look algorithm decid\n",
      "whether point insid space defin point space defin\n",
      "point mean space defin close mesh defin point like\n",
      "simpl exampl side parallel problem side non\n",
      "parallel opposit side must size etc pleas mail\n",
      "inform  thank great effort  oliv   name\n",
      "kretzschmar inst ike univers stuttgart phone\n",
      "pfaffenwaldr fax stuttgart\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "               defin,  2482,  0.506\n",
      "           stuttgart,  8478,  0.373\n",
      "               point,  6710,  0.325\n",
      "               space,  8164,  0.262\n",
      "            parallel,  6420,  0.241\n",
      "               insid,  4671,  0.216\n",
      "           algorithm,   407,  0.200\n",
      "               decid,  2442,  0.200\n",
      "                inst,  4677,  0.175\n",
      "                oliv,  6223,  0.159\n",
      "                mesh,  5648,  0.148\n",
      "             opposit,  6270,  0.121\n",
      "                 hey,  4283,  0.118\n",
      "               plane,  6663,  0.112\n",
      "              effort,  3007,  0.108\n",
      "                size,  8000,  0.107\n"
     ]
    }
   ],
   "source": [
    "m = findMismatches(2, 9, data)\n",
    "m\n",
    "print(\"\")\n",
    "printDoc(m[0], data, km, cluster_terms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cluster 1/Category 3\n",
    "\n",
    "Now, one more Let's look at cluster 1 (moral object valu say christian kuiper smiley subject peopl atom), where most docs came from category 3 (talk.religion.misc)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Document ID: 61\n",
      "Filename: 61067\n",
      "Truth category: 2 (sci.space)\n",
      "Assigned cluster: 1 (moral object valu say christian kuiper smiley subject peopl)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: chico@ccsun.unicamp.br (Francisco da Fonseca Rodrigues)\n",
      "Subject: New planet/Kuiper object found?\n",
      "X-Added: Forwarded by Space Digest\n",
      "Organization: [via International Space University]\n",
      "Original-Sender: isu@VACATION.VENARI.CS.CMU.EDU\n",
      "Distribution: sci\n",
      "Lines: 28\n",
      "\n",
      "\n",
      "\tTonigth a TV journal here in Brasil announced that an object,\n",
      "beyond Pluto's orbit, was found by an observatory at Hawaii. They\n",
      "named the object Karla.\n",
      "\n",
      "\tThe program said the object wasn't a gaseous giant planet, and\n",
      "should be composed by rocks and ices.\n",
      "\n",
      "\tCan someone confirm these information? Could this object be a\n",
      "new planet or a Kuiper object?\n",
      "\n",
      "\tThanks in advance.\n",
      "\n",
      "\tFrancisco.\n",
      "\n",
      "-----------------------=====================================----the stars,----\n",
      "|    ._,               | Francisco da Fonseca Rodrigues    |       o   o     |\n",
      "|  ,_| |._/\\           |                                   |     o         o |\n",
      "|  |       |o/^^~-._   | COTUCA-Colegio Tecnico da UNICAMP |   o             |\n",
      "|/-'    BRASIL      | ~|                                   |  o      o o     |\n",
      "|\\__/|_            /'  | Depto de Processamento de Dados   |  o    o  o  o   |\n",
      "|      \\__  Cps   | .  |                                   |   o  o  o    o  |\n",
      "|        |   * __/'    | InterNet : chico@ccsun.unicamp.br |     o o      o  |\n",
      "|        >   /'        |            cotuca@ccvax.unicamp.br|             o   |\n",
      "|      /'   /'         | Fone/Fax : 55-0192-32-9519        | o         o     |\n",
      "|     ~~^\\/'           | Campinas - SP - Brasil            |    o   o        |\n",
      "-----------------------=====================================----like dust.----\n",
      "\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " new planet kuiper object found ad forward space\n",
      "digest       tonigth journal brasil announc object\n",
      "beyond pluto orbit found observatori hawaii name\n",
      "object karla  program said object wasn't gaseous\n",
      "giant planet compos rock ice  someon confirm\n",
      "inform could object new planet kuiper object\n",
      "thank advanc  francisco  star francisco fonseca\n",
      "rodrigu  cotuca colegio tecnico unicamp brasil\n",
      "depto processamento dado cps   fone fax campina\n",
      "brasil like dust\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "              brasil,  1273,  0.497\n",
      "              object,  6173,  0.453\n",
      "              planet,  6664,  0.282\n",
      "              kuiper,  5080,  0.268\n",
      "           francisco,  3652,  0.232\n",
      "                 cps,  2199,  0.160\n",
      "             gaseous,  3782,  0.155\n",
      "                fone,  3581,  0.155\n",
      "               karla,  4958,  0.141\n",
      "              hawaii,  4208,  0.136\n",
      "              compos,  1981,  0.127\n",
      "                 ice,  4475,  0.124\n",
      "                dust,  2928,  0.121\n",
      "             confirm,  2032,  0.120\n",
      "                 new,  6023,  0.119\n",
      "             journal,  4898,  0.116\n",
      "Document ID: 551\n",
      "Filename: 61150\n",
      "Truth category: 2 (sci.space)\n",
      "Assigned cluster: 1 (moral object valu say christian kuiper smiley subject peopl)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: jafoust@cco.caltech.edu (Jeff Foust)\n",
      "Subject: Re: New planet/Kuiper object found?\n",
      "Organization: Caltech:  Pasadena, California, USA\n",
      "Lines: 12\n",
      "Distribution: sci\n",
      "NNTP-Posting-Host: sandman.caltech.edu\n",
      "\n",
      "In a recent article jdnicoll@prism.ccs.uwo.ca (James Davis Nicoll) writes:\n",
      ">\tIf the  new  Kuiper belt object *is*  called 'Karla', the next\n",
      ">one  should be called 'Smiley'.\n",
      "\n",
      "Unless I'm imaging things, (always a possibility =) 1992 QB1, the Kuiper Belt\n",
      "object discovered last year, is known as Smiley.\n",
      "\n",
      "-- \n",
      "Jeff Foust              [49 days!]\t\"You're from outer space.\"\n",
      "Senior, Planetary Science, Caltech\t\"No, I'm from Iowa.  I only work in\n",
      "jafoust@cco.caltech.edu\t\t\t outer space.\"\n",
      "jeff@scn1.jpl.nasa.gov\t\t\t-- from Star Trek IV: The Voyage Home\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " new planet kuiper object found       new kuiper\n",
      "belt object call karla next one call smiley\n",
      "unless i'm imag thing alway possibl qb1 kuiper\n",
      "belt object discov last year known smiley   jeff\n",
      "foust day you'r outer space senior planetari\n",
      "scienc caltech i'm iowa work\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "              kuiper,  5080,  0.531\n",
      "              smiley,  8063,  0.330\n",
      "                belt,  1017,  0.315\n",
      "              object,  6173,  0.299\n",
      "               foust,  3631,  0.211\n",
      "                 qb1,  7005,  0.192\n",
      "               karla,  4958,  0.186\n",
      "                iowa,  4754,  0.179\n",
      "             caltech,  1468,  0.173\n",
      "              senior,  7787,  0.170\n",
      "                 new,  6023,  0.158\n",
      "               outer,  6324,  0.156\n",
      "                jeff,  4839,  0.144\n",
      "              discov,  2698,  0.141\n",
      "           planetari,  6665,  0.138\n",
      "              planet,  6664,  0.124\n",
      "Document ID: 1161\n",
      "Filename: 61138\n",
      "Truth category: 2 (sci.space)\n",
      "Assigned cluster: 1 (moral object valu say christian kuiper smiley subject peopl)\n",
      "Raw data:\n",
      "============ START RAW ==========================\n",
      "\n",
      "From: jdnicoll@prism.ccs.uwo.ca (James Davis Nicoll)\n",
      "Subject: Re: New planet/Kuiper object found?\n",
      "Organization: University of Western Ontario, London\n",
      "Distribution: sci\n",
      "Nntp-Posting-Host: prism.engrg.uwo.ca\n",
      "Lines: 5\n",
      "\n",
      "\tIf the  new  Kuiper belt object *is*  called 'Karla', the next\n",
      "one  should be called 'Smiley'.\n",
      "\n",
      "\t\t\t\t\t\t\tJames Nicoll\n",
      "\n",
      "\n",
      "\n",
      "========== END RAW ============================\n",
      "\n",
      "Preprocessed:\n",
      "\n",
      "============ START PREPROCESSED =================\n",
      "\n",
      " new planet kuiper object found      new kuiper\n",
      "belt object call karla next one call smiley  jame\n",
      "nicol\n",
      "========== END PREPROCESSED =====================\n",
      "\n",
      "\n",
      "Feature vector (Top 15, sorted, non-zero values only):\n",
      "Term, Index, Value:\n",
      "              kuiper,  5080,  0.600\n",
      "              object,  6173,  0.338\n",
      "               nicol,  6055,  0.332\n",
      "               karla,  4958,  0.316\n",
      "              smiley,  8063,  0.279\n",
      "                 new,  6023,  0.267\n",
      "                belt,  1017,  0.267\n",
      "                jame,  4820,  0.227\n",
      "              planet,  6664,  0.210\n"
     ]
    }
   ],
   "source": [
    "m = findMismatches(3, 1, data)\n",
    "m\n",
    "print(\"\")\n",
    "printDoc(m[0], data, km, cluster_terms)\n",
    "printDoc(m[1], data, km, cluster_terms)\n",
    "printDoc(m[3], data, km, cluster_terms)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}