nbs/imdb_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.datasets import imdb\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "import keras.layers as kl\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, Trials\n",
    "from kopt import CompileFN, KMongoTrials, test_fn\n",
    "\n",
    "# 1. define the data function returning training, (validation, test) data\n",
    "def data(max_features=5000, maxlen=80):\n",
    "    (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)\n",
    "    x_train = sequence.pad_sequences(x_train, maxlen=maxlen)\n",
    "    x_test = sequence.pad_sequences(x_test, maxlen=maxlen)\n",
    "    return (x_train[:100], y_train[:100], max_features), (x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Define the model function returning a compiled Keras model\n",
    "def model(train_data, lr=0.001,\n",
    "          embedding_dims=128, rnn_units=64,\n",
    "          dropout=0.2):\n",
    "    # extract data dimensions\n",
    "    max_features = train_data[2]\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(kl.Embedding(max_features, embedding_dims))\n",
    "    model.add(kl.LSTM(rnn_units, dropout=dropout, recurrent_dropout=dropout))\n",
    "    model.add(kl.Dense(1, activation='sigmoid'))\n",
    "\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer=Adam(lr=lr),\n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the objective function\n",
    "db_name = \"imdb\"\n",
    "exp_name = \"myexp1\"\n",
    "\n",
    "objective = CompileFN(db_name, exp_name,\n",
    "                      data_fn=data,\n",
    "                      model_fn=model,\n",
    "                      loss_metric=\"acc\", # which metric to optimize for\n",
    "                      loss_metric_mode=\"max\",  # try to maximize the metric\n",
    "                      valid_split=.2, # use 20% of the training data for the validation set\n",
    "                      save_model='best', # checkpoint the best model\n",
    "                      save_results=True, # save the results as .json (in addition to mongoDB)\n",
    "                      save_dir=\"./saved_models/\")  # place to store the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the hyper-parameter ranges\n",
    "# see https://github.com/hyperopt/hyperopt/wiki/FMin for more info\n",
    "hyper_params = {\n",
    "    \"data\": {\n",
    "        \"max_features\": 100,\n",
    "        \"maxlen\": 80,\n",
    "    },\n",
    "    \"model\": {\n",
    "        \"lr\": hp.loguniform(\"m_lr\", np.log(1e-4), np.log(1e-2)), # 0.0001 - 0.01\n",
    "        \"embedding_dims\": hp.choice(\"m_emb\", (64, 128)),\n",
    "        \"rnn_units\": 64,\n",
    "        \"dropout\": hp.uniform(\"m_do\", 0, 0.5),\n",
    "},\n",
    "    \"fit\": {\n",
    "        \"epochs\": 20\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-10-07 17:53:55,100 [INFO] Load data...\n",
      "2017-10-07 17:54:00,418 [INFO] Fit...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['loss', 'acc']\n",
      "Train on 80 samples, validate on 20 samples\n",
      "Epoch 1/1\n",
      "0s - loss: 0.6931 - acc: 0.5125 - val_loss: 0.6699 - val_acc: 0.7000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-10-07 17:54:03,808 [INFO] Evaluate...\n",
      "2017-10-07 17:54:04,060 [INFO] Done!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Returned value:\n",
      "{'eval': {'acc': 0.69999998807907104, 'loss': 0.66988599300384521},\n",
      " 'history': {'loss': {'acc': [0.51249999999999996],\n",
      "                      'epoch': [0],\n",
      "                      'loss': [0.6931055903434753],\n",
      "                      'val_acc': [0.69999998807907104],\n",
      "                      'val_loss': [0.66988599300384521]},\n",
      "             'params': {'batch_size': 32,\n",
      "                        'do_validation': True,\n",
      "                        'epochs': 1,\n",
      "                        'metrics': ['loss', 'acc', 'val_loss', 'val_acc'],\n",
      "                        'samples': 80,\n",
      "                        'steps': None,\n",
      "                        'verbose': 2}},\n",
      " 'loss': -0.69999998807907104,\n",
      " 'name': {'data': 'data',\n",
      "          'model': 'model',\n",
      "          'optim_metric': 'acc',\n",
      "          'optim_metric_mode': 'acc'},\n",
      " 'param': {'data': {'max_features': 100, 'maxlen': 80},\n",
      "           'fit': {'batch_size': 32,\n",
      "                   'early_stop_monitor': 'val_loss',\n",
      "                   'epochs': 1,\n",
      "                   'patience': 10},\n",
      "           'model': {'dropout': 0.31760152349994536,\n",
      "                     'embedding_dims': 64,\n",
      "                     'lr': 0.0035943038525379817,\n",
      "                     'rnn_units': 64}},\n",
      " 'path': {'model': '/tmp/kopt_test//imdb/myexp1//train_models/5dbfcb40-28c9-4aa3-9176-178ac5c3e85f.h5',\n",
      "          'results': '/tmp/kopt_test//imdb/myexp1//train_models/5dbfcb40-28c9-4aa3-9176-178ac5c3e85f.json'},\n",
      " 'status': 'ok',\n",
      " 'time': {'duration': {'dataload': 4.920822,\n",
      "                       'total': 8.960585,\n",
      "                       'training': 4.039763},\n",
      "          'end': '2017-10-07 17:54:04.059751',\n",
      "          'start': '2017-10-07 17:53:55.099166'}}\n"
     ]
    }
   ],
   "source": [
    "# test model training, on a small subset for one epoch\n",
    "test_fn(objective, hyper_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-10-07 17:54:08,950 [INFO] tpe_transform took 0.002530 seconds\n",
      "2017-10-07 17:54:08,951 [INFO] TPE using 0 trials\n",
      "2017-10-07 17:54:08,954 [INFO] Load data...\n",
      "2017-10-07 17:54:13,933 [INFO] Fit...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['loss', 'acc']\n",
      "Train on 80 samples, validate on 20 samples\n",
      "Epoch 1/20\n",
      "0s - loss: 0.6929 - acc: 0.5125 - val_loss: 0.6252 - val_acc: 0.7000\n",
      "Epoch 2/20\n",
      "0s - loss: 0.6853 - acc: 0.5500 - val_loss: 0.6495 - val_acc: 0.6500\n",
      "Epoch 3/20\n",
      "0s - loss: 0.6691 - acc: 0.6500 - val_loss: 0.6532 - val_acc: 0.7000\n",
      "Epoch 4/20\n",
      "0s - loss: 0.6291 - acc: 0.7500 - val_loss: 0.5904 - val_acc: 0.6000\n",
      "Epoch 5/20\n",
      "0s - loss: 0.5450 - acc: 0.7375 - val_loss: 0.6378 - val_acc: 0.7000\n",
      "Epoch 6/20\n",
      "0s - loss: 0.4470 - acc: 0.7625 - val_loss: 0.6246 - val_acc: 0.7000\n",
      "Epoch 7/20\n",
      "0s - loss: 0.5676 - acc: 0.7500 - val_loss: 0.6697 - val_acc: 0.7500\n",
      "Epoch 8/20\n",
      "0s - loss: 0.3634 - acc: 0.8625 - val_loss: 0.7863 - val_acc: 0.6000\n",
      "Epoch 9/20\n",
      "0s - loss: 0.3516 - acc: 0.8625 - val_loss: 0.8624 - val_acc: 0.6500\n",
      "Epoch 10/20\n",
      "0s - loss: 0.2364 - acc: 0.9125 - val_loss: 0.7997 - val_acc: 0.5000\n",
      "Epoch 11/20\n",
      "0s - loss: 0.2183 - acc: 0.9500 - val_loss: 1.0836 - val_acc: 0.4500\n",
      "Epoch 12/20\n",
      "0s - loss: 0.1658 - acc: 0.9500 - val_loss: 1.1220 - val_acc: 0.4500\n",
      "Epoch 13/20\n",
      "0s - loss: 0.1018 - acc: 0.9750 - val_loss: 1.4189 - val_acc: 0.5000\n",
      "Epoch 14/20\n",
      "0s - loss: 0.1085 - acc: 0.9625 - val_loss: 1.3545 - val_acc: 0.5500\n",
      "Epoch 15/20\n",
      "0s - loss: 0.2012 - acc: 0.9375 - val_loss: 1.2497 - val_acc: 0.5500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-10-07 17:54:21,912 [INFO] Evaluate...\n",
      "2017-10-07 17:54:22,468 [INFO] Done!\n",
      "2017-10-07 17:54:22,474 [INFO] tpe_transform took 0.002084 seconds\n",
      "2017-10-07 17:54:22,475 [INFO] TPE using 1/1 trials with best loss -0.600000\n",
      "2017-10-07 17:54:22,479 [INFO] Load data...\n",
      "2017-10-07 17:54:27,459 [INFO] Fit...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['loss', 'acc']\n",
      "Train on 80 samples, validate on 20 samples\n",
      "Epoch 1/20\n",
      "1s - loss: 0.6927 - acc: 0.5500 - val_loss: 0.6739 - val_acc: 0.7000\n",
      "Epoch 2/20\n",
      "0s - loss: 0.6863 - acc: 0.5500 - val_loss: 0.6485 - val_acc: 0.7000\n",
      "Epoch 3/20\n",
      "0s - loss: 0.6751 - acc: 0.5500 - val_loss: 0.6320 - val_acc: 0.7000\n",
      "Epoch 4/20\n",
      "0s - loss: 0.6750 - acc: 0.5500 - val_loss: 0.6221 - val_acc: 0.7000\n",
      "Epoch 5/20\n",
      "0s - loss: 0.6869 - acc: 0.5500 - val_loss: 0.6367 - val_acc: 0.7000\n",
      "Epoch 6/20\n",
      "0s - loss: 0.6581 - acc: 0.5750 - val_loss: 0.6323 - val_acc: 0.6500\n",
      "Epoch 7/20\n",
      "0s - loss: 0.6562 - acc: 0.6000 - val_loss: 0.6239 - val_acc: 0.6500\n",
      "Epoch 8/20\n",
      "0s - loss: 0.6367 - acc: 0.6125 - val_loss: 0.5990 - val_acc: 0.6500\n",
      "Epoch 9/20\n",
      "0s - loss: 0.6026 - acc: 0.7000 - val_loss: 0.6150 - val_acc: 0.7500\n",
      "Epoch 10/20\n",
      "0s - loss: 0.5760 - acc: 0.7875 - val_loss: 0.5984 - val_acc: 0.8000\n",
      "Epoch 11/20\n",
      "0s - loss: 0.5383 - acc: 0.8125 - val_loss: 0.6042 - val_acc: 0.7500\n",
      "Epoch 12/20\n",
      "0s - loss: 0.5136 - acc: 0.8000 - val_loss: 0.5976 - val_acc: 0.6500\n",
      "Epoch 13/20\n",
      "0s - loss: 0.4534 - acc: 0.7875 - val_loss: 0.6000 - val_acc: 0.7000\n",
      "Epoch 14/20\n",
      "0s - loss: 0.4166 - acc: 0.8625 - val_loss: 0.6372 - val_acc: 0.6500\n",
      "Epoch 15/20\n",
      "0s - loss: 0.3760 - acc: 0.8625 - val_loss: 0.6083 - val_acc: 0.7000\n",
      "Epoch 16/20\n",
      "0s - loss: 0.3851 - acc: 0.8375 - val_loss: 0.6039 - val_acc: 0.7000\n",
      "Epoch 17/20\n",
      "0s - loss: 0.3941 - acc: 0.8125 - val_loss: 0.6135 - val_acc: 0.7500\n",
      "Epoch 18/20\n",
      "0s - loss: 0.4082 - acc: 0.7875 - val_loss: 0.6570 - val_acc: 0.7000\n",
      "Epoch 19/20\n",
      "0s - loss: 0.3180 - acc: 0.9125 - val_loss: 0.6680 - val_acc: 0.6500\n",
      "Epoch 20/20\n",
      "0s - loss: 0.3227 - acc: 0.8875 - val_loss: 0.6794 - val_acc: 0.6500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-10-07 17:54:37,040 [INFO] Evaluate...\n",
      "2017-10-07 17:54:37,782 [INFO] Done!\n"
     ]
    }
   ],
   "source": [
    "# run hyper-parameter optimization sequentially (without any database)\n",
    "trials = Trials()\n",
    "best = fmin(objective, hyper_params, trials=trials, algo=tpe.suggest, max_evals=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distributed workers - (K)MongoTrials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting mongod ['mongod', '--dbpath=/tmp/tmp7e0lvesw', '--noprealloc', '--port=22334']\n"
     ]
    }
   ],
   "source": [
    "import tempfile\n",
    "import os\n",
    "import subprocess\n",
    "# Start the mongodb database and a worker\n",
    "mongodb_path = tempfile.mkdtemp()\n",
    "results_path = tempfile.mkdtemp()\n",
    "\n",
    "proc_args = [\"mongod\",\n",
    "             \"--dbpath=%s\" % mongodb_path,\n",
    "             \"--noprealloc\",\n",
    "             \"--port=22334\"]\n",
    "print(\"starting mongod\", proc_args)\n",
    "mongodb_proc = subprocess.Popen(\n",
    "        proc_args,\n",
    "        cwd=mongodb_path,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start the worker\n",
    "from kopt.utils import merge_dicts\n",
    "proc_args_worker = [\"hyperopt-mongo-worker\",\n",
    "                    \"--mongo=localhost:22334/imdb\",\n",
    "                    \"--poll-interval=0.1\"]\n",
    "\n",
    "mongo_worker_proc = subprocess.Popen(\n",
    "    proc_args_worker,\n",
    "    env=merge_dicts(os.environ, {\"PYTHONPATH\": os.getcwd()}),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "## In Order for pickling of the functions to work,\n",
    "## we need to import the functions from a module different\n",
    "## than __main___\n",
    "## I've implemented them in model.py and data.py\n",
    "import model\n",
    "import data\n",
    "objective.data_fn = data.data\n",
    "objective.model_fn = model.model\n",
    "objective.save_dir = results_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-10-07 18:26:16,861 [INFO] PROTOCOL mongo\n",
      "2017-10-07 18:26:16,863 [INFO] USERNAME None\n",
      "2017-10-07 18:26:16,865 [INFO] HOSTNAME localhost\n",
      "2017-10-07 18:26:16,866 [INFO] PORT 22334\n",
      "2017-10-07 18:26:16,867 [INFO] PATH /imdb/jobs\n",
      "2017-10-07 18:26:16,868 [INFO] DB imdb\n",
      "2017-10-07 18:26:16,870 [INFO] COLLECTION jobs\n",
      "2017-10-07 18:26:16,871 [INFO] PASS None\n",
      "2017-10-07 18:26:17,305 [WARNING] no last_id found, re-trying\n",
      "2017-10-07 18:26:18,315 [INFO] tpe_transform took 0.002277 seconds\n",
      "2017-10-07 18:26:18,316 [INFO] TPE using 0 trials\n",
      "2017-10-07 18:26:19,336 [INFO] tpe_transform took 0.003541 seconds\n",
      "2017-10-07 18:26:19,338 [INFO] TPE using 1/1 trials with best loss inf\n",
      "2017-10-07 18:26:20,348 [INFO] Waiting for 2 jobs to finish ...\n",
      "2017-10-07 18:26:58,425 [INFO] Queue empty, exiting run.\n"
     ]
    }
   ],
   "source": [
    "# run hyper-parameter optimization in parallel (saving the results to MonogoDB)\n",
    "# Follow the hyperopt guide:\n",
    "# https://github.com/hyperopt/hyperopt/wiki/Parallelizing-Evaluations-During-Search-via-MongoDB\n",
    "# KMongoTrials extends hyperopt.MongoTrials with convenience methods\n",
    "trials = KMongoTrials(db_name, exp_name,\n",
    "                      ip=\"localhost\",\n",
    "                      port=22334)\n",
    "best = fmin(objective, hyper_params, trials=trials, algo=tpe.suggest, max_evals=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analyze the trials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of submitted trials\n",
    "len(trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>loss</th>\n",
       "      <th>status</th>\n",
       "      <th>eval.acc</th>\n",
       "      <th>eval.loss</th>\n",
       "      <th>name.data</th>\n",
       "      <th>name.model</th>\n",
       "      <th>name.optim_metric</th>\n",
       "      <th>name.optim_metric_mode</th>\n",
       "      <th>param.data.max_features</th>\n",
       "      <th>param.data.maxlen</th>\n",
       "      <th>param.fit.batch_size</th>\n",
       "      <th>param.fit.early_stop_monitor</th>\n",
       "      <th>param.fit.epochs</th>\n",
       "      <th>param.fit.patience</th>\n",
       "      <th>param.model.dropout</th>\n",
       "      <th>param.model.embedding_dims</th>\n",
       "      <th>param.model.lr</th>\n",
       "      <th>param.model.rnn_units</th>\n",
       "      <th>path.model</th>\n",
       "      <th>path.results</th>\n",
       "      <th>time.duration.dataload</th>\n",
       "      <th>time.duration.total</th>\n",
       "      <th>time.duration.training</th>\n",
       "      <th>time.end</th>\n",
       "      <th>time.start</th>\n",
       "      <th>n_epoch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>-0.75</td>\n",
       "      <td>ok</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.5620</td>\n",
       "      <td>data</td>\n",
       "      <td>model</td>\n",
       "      <td>acc</td>\n",
       "      <td>acc</td>\n",
       "      <td>100</td>\n",
       "      <td>80</td>\n",
       "      <td>32</td>\n",
       "      <td>val_loss</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.0161</td>\n",
       "      <td>64</td>\n",
       "      <td>0.0083</td>\n",
       "      <td>64</td>\n",
       "      <td>/tmp/tmp2cmnwyi8/imdb...</td>\n",
       "      <td>/tmp/tmp2cmnwyi8/imdb...</td>\n",
       "      <td>5.7646</td>\n",
       "      <td>18.6963</td>\n",
       "      <td>12.9317</td>\n",
       "      <td>2017-10-07 18:26:57.1...</td>\n",
       "      <td>2017-10-07 18:26:38.4...</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>-0.80</td>\n",
       "      <td>ok</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.6083</td>\n",
       "      <td>data</td>\n",
       "      <td>model</td>\n",
       "      <td>acc</td>\n",
       "      <td>acc</td>\n",
       "      <td>100</td>\n",
       "      <td>80</td>\n",
       "      <td>32</td>\n",
       "      <td>val_loss</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>0.3088</td>\n",
       "      <td>64</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>64</td>\n",
       "      <td>/tmp/tmp2cmnwyi8/imdb...</td>\n",
       "      <td>/tmp/tmp2cmnwyi8/imdb...</td>\n",
       "      <td>5.5815</td>\n",
       "      <td>18.7385</td>\n",
       "      <td>13.1570</td>\n",
       "      <td>2017-10-07 18:26:58.3...</td>\n",
       "      <td>2017-10-07 18:26:39.6...</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tid  loss status  eval.acc  eval.loss name.data name.model name.optim_metric  \\\n",
       "0    1 -0.75     ok      0.75     0.5620      data      model               acc   \n",
       "1    2 -0.80     ok      0.80     0.6083      data      model               acc   \n",
       "\n",
       "  name.optim_metric_mode  param.data.max_features  param.data.maxlen  param.fit.batch_size  \\\n",
       "0                    acc                      100                 80                    32   \n",
       "1                    acc                      100                 80                    32   \n",
       "\n",
       "  param.fit.early_stop_monitor  param.fit.epochs  param.fit.patience  param.model.dropout  \\\n",
       "0                  val_loss                   20                  10               0.0161   \n",
       "1                  val_loss                   20                  10               0.3088   \n",
       "\n",
       "   param.model.embedding_dims  param.model.lr  param.model.rnn_units                path.model  \\\n",
       "0                        64            0.0083                     64  /tmp/tmp2cmnwyi8/imdb...   \n",
       "1                        64            0.0009                     64  /tmp/tmp2cmnwyi8/imdb...   \n",
       "\n",
       "               path.results  time.duration.dataload  time.duration.total  time.duration.training  \\\n",
       "0  /tmp/tmp2cmnwyi8/imdb...                  5.7646              18.6963                 12.9317   \n",
       "1  /tmp/tmp2cmnwyi8/imdb...                  5.5815              18.7385                 13.1570   \n",
       "\n",
       "                   time.end                time.start  n_epoch  \n",
       "0  2017-10-07 18:26:57.1...  2017-10-07 18:26:38.4...       18  \n",
       "1  2017-10-07 18:26:58.3...  2017-10-07 18:26:39.6...       19  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ALl the traial information in one tidy pd.DataFrame\n",
    "trials.as_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.models.Sequential at 0x7fae6023ef28>"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load the best model\n",
    "model = trials.load_model(trials.best_trial_tid())\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>epoch</th>\n",
       "      <th>acc</th>\n",
       "      <th>loss</th>\n",
       "      <th>val_acc</th>\n",
       "      <th>val_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5375</td>\n",
       "      <td>0.6933</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.6887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5625</td>\n",
       "      <td>0.6898</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.6781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.5500</td>\n",
       "      <td>0.6864</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.6696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>0.7000</td>\n",
       "      <td>0.6345</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.6209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>0.6750</td>\n",
       "      <td>0.6151</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2</td>\n",
       "      <td>19</td>\n",
       "      <td>0.7375</td>\n",
       "      <td>0.5972</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6119</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    tid  epoch     acc    loss  val_acc  val_loss\n",
       "0     2      0  0.5375  0.6933      0.7    0.6887\n",
       "1     2      1  0.5625  0.6898      0.7    0.6781\n",
       "2     2      2  0.5500  0.6864      0.7    0.6696\n",
       "..  ...    ...     ...     ...      ...       ...\n",
       "17    2     17  0.7000  0.6345      0.7    0.6209\n",
       "18    2     18  0.6750  0.6151      0.8    0.6083\n",
       "19    2     19  0.7375  0.5972      0.8    0.6119\n",
       "\n",
       "[20 rows x 6 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see the training history of the best model\n",
    "train_hist = trials.train_history(trials.best_trial_tid())\n",
    "train_hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# close the processes (mongodb & hyperopt worker)\n",
    "mongo_worker_proc.kill()\n",
    "mongodb_proc.kill()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}