DQN-Tensorflow-Movie.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DQN\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training of DQN in Tensorflow\n",
    "--------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/matplotlib/__init__.py:1401: UserWarning:  This call to matplotlib.use() has no effect\n",
      "because the backend has already been chosen;\n",
      "matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n",
      "or matplotlib.backends is imported for the first time.\n",
      "\n",
      "  warnings.warn(_use_error_msg)\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "from util import *\n",
    "from dlg_manager import *\n",
    "from alg import *\n",
    "from agent import *\n",
    "from user_sim import *\n",
    "from state_tracker import *\n",
    "import random\n",
    "from config import *\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from nlg import *\n",
    "from six.moves import cPickle as pickle\n",
    "import IPython\n",
    "import copy, argparse, json\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Action Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keys = 11\n",
      "Sample of dict:\n",
      "- closing: 5\n",
      "- request: 0\n",
      "- confirm_answer: 3\n",
      "- thanks: 7\n",
      "- inform: 1\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "act_set_path = './data/dia_acts.txt'\n",
    "act_set = text_to_dict(act_set_path)\n",
    "sample_dict(act_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### slot set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keys = 29\n",
      "Sample of dict:\n",
      "- state: 21\n",
      "- mc_list: 28\n",
      "- description: 6\n",
      "- critic_rating: 4\n",
      "- date: 5\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "slots_set_path = \"./data/slot_set.txt\"\n",
    "slot_set = text_to_dict(slots_set_path)\n",
    "sample_dict(slot_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### movie dic: info about movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keys = 991\n",
      "Sample of dict:\n",
      "- 915: {'state': 'california', 'moviename': '10 cloverfield lane', 'starttime': '10:05', 'city': 'tulare', 'theater': 'regal visalia stadium 10', 'date': 'this weekend'}\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "movie_kb_path = \"./data/movie_kb.1k.p\"\n",
    "movie_kb = pickle.load(open(movie_kb_path, 'rb'), encoding=\"latin\")\n",
    "sample_dict(movie_kb, sample_size=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Language Generator (pretrained)\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#\n",
    "nlg_model_path = './data/trained_model/nlg/lstm_tanh_relu_[1468202263.38]_2_0.610.p'\n",
    "nlg_model = Nlg()\n",
    "nlg_model.load_nlg_model(nlg_model_path)\n",
    "diaact_nl_pairs_path = \"./data/nlg/dia_act_nl_pairs.v6.json\"\n",
    "nlg_model.load_predefine_act_nl_pairs(diaact_nl_pairs_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_path: .\\data\\movieMultiLine.Annot.Corrected.Final.v3.csv\n",
      "max_epochs: 200\n",
      "cv_fold: 6\n",
      "momentum: 0.1\n",
      "dia_slot_val: 2\n",
      "trained_model_path: None\n",
      "eva_metric: 2\n",
      "hidden_size: 100\n",
      "write_model_dir: .\\checkpoints\\template\\07102016\\\n",
      "reg_cost: 0.001\n",
      "split_method: 1\n",
      "valid_test: 0\n",
      "model: lstm_tanh\n",
      "batch_size: 16\n",
      "sdgtype: rmsprop\n",
      "pretrained_model_path: None\n",
      "learning_rate: 0.001\n",
      "smooth_eps: 1e-08\n",
      "feed_recurrence: 0\n",
      "save_check_point: 20\n",
      "init_rnn: 0\n",
      "act_set: data/dia_acts.txt\n",
      "decay_rate: 0.999\n",
      "grad_clip: -0.0001\n",
      "check_point: 20\n",
      "slot_set: data/slot_set.txt\n",
      "slot_rep: 1\n",
      "activation_func: relu\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "model_params = pickle.load(open(nlg_model_path, 'rb'), encoding='latin1')\n",
    "params = model_params['params']\n",
    "params['batch_size'] = 16\n",
    "batch_size = 16\n",
    "save_check_point = 20\n",
    "params['trained_model_path'] = None\n",
    "for k in params:\n",
    "    print(\"{}: {}\".format(k, params[k]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# User Simulator\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### goal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "goals length: 128\n",
      "Sample the first goal: \n",
      "{'request_slots': {}, 'inform_slots': {'state': 'al', 'numberofpeople': '1', 'starttime': 'around 2pm', 'city': 'birmingham', 'theater': 'carmike summit 16', 'moviename': 'zootopia', 'date': 'today'}, 'diaact': 'request'}\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "goal_file_path = './data/user_goals_first_turn_template.part.movie.v1.p'\n",
    "all_goal_set = pickle.load(open(goal_file_path, 'rb'), encoding=\"latin\")\n",
    "print(\"goals length: {}\".format(len(all_goal_set)))\n",
    "print(\"Sample the first goal: \\n{}\".format(all_goal_set[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split goal set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "102\n",
      "0\n",
      "26\n",
      "128\n"
     ]
    }
   ],
   "source": [
    "# split goal set\n",
    "split_fold = params.get('split_fold', 5)\n",
    "goal_set = {'train':[], 'valid':[], 'test':[], 'all':[]}\n",
    "for u_goal_id, u_goal in enumerate(all_goal_set):\n",
    "    if u_goal_id % split_fold == 1: goal_set['test'].append(u_goal)\n",
    "    else: goal_set['train'].append(u_goal)\n",
    "    goal_set['all'].append(u_goal)\n",
    "print(len(goal_set['train']))\n",
    "print(len(goal_set['valid']))\n",
    "print(len(goal_set['test']))\n",
    "print(len(goal_set['all']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### user simulator param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#\n",
    "usersim_params = {}\n",
    "usersim_params['max_turn'] = 40\n",
    "usersim_params['slot_err_prob'] = 0.00\n",
    "# slot_err_mode: 0 for slot_val only; 1 for three errs\n",
    "usersim_params['slot_err_mode'] = 0\n",
    "usersim_params['intent_err_prob'] = 0\n",
    "# run_mode: 0 for default NL; 1 for dia_act; 2 for both\n",
    "usersim_params['run_mode'] = 0\n",
    "# 0 for dia_act level; 1 for NL level\n",
    "usersim_params['act_level'] = 0\n",
    "# train/test/all; default is all\n",
    "usersim_params['learn_phase'] = 'all'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a movie dictionary for user simulator - slot:possible values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keys = 20\n",
      "Sample of dict:\n",
      "- greeting: ['hey', 'hello', 'good morning', 'hi', 'there', 'hello there', 'happy to', 'good evening', 'hi welcome', 'hey there', 'hi there']\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "movie_dict_path = './data/user/dicts.v3.p'\n",
    "movie_dictionary = pickle.load(open(movie_dict_path, 'rb'), encoding=\"latin\")\n",
    "samples = sample_dict(movie_dictionary, sample_size=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Create a User"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "user = RuleSimulator(movie_dictionary, act_set, slot_set, goal_set, usersim_params)\n",
    "user.set_nlg_model(nlg_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agent\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pretrained model path = None\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "agent_params = {}\n",
    "# maximum length of each dialog (default=20, 0=no maximum length)\n",
    "agent_params['max_turn'] = 40\n",
    "# Epsilon to determine stochasticity of epsilon-greedy agent policies\n",
    "agent_params['epsilon'] = 0\n",
    "# run_mode: 0 for default NL; 1 for dia_act; 2 for both\n",
    "agent_params['agent_run_mode'] = 3\n",
    "# 0 for dia_act level; 1 for NL level\n",
    "agent_params['agent_act_level'] = 0\n",
    "\n",
    "############### DQN #################\n",
    "# the size for experience replay\n",
    "agent_params['experience_replay_pool_size'] = 10000\n",
    "# # the hidden size for DQN\n",
    "agent_params['dqn_hidden_size'] = 60\n",
    "agent_params['batch_size'] = 16\n",
    "# # gamma for DQN\n",
    "agent_params['gamma'] = 0.9\n",
    "# # predict model for DQN\n",
    "agent_params['predict_mode'] = True\n",
    "agent_params['trained_model_path'] = params['pretrained_model_path']\n",
    "#####################################\n",
    "print(\"pretrained model path = {}\".format(agent_params['trained_model_path']))\n",
    "# 0: no warm start; 1: warm start for training\n",
    "agent_params['warm_start'] = 1\n",
    "# run_mode: 0 for NL; 1 for dia_act\n",
    "agent_params['cmd_input_mode'] = 0\n",
    "\n",
    "success_rate_threshold = 0.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create an agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "code_folding": [],
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "act_cardinality  11\n",
      "feasible_actions 43\n",
      "80\n"
     ]
    }
   ],
   "source": [
    "# agent = RequestBasicsAgent(movie_kb, act_set, slot_set, agent_params)\n",
    "# agent = AgentDQN(movie_kb, act_set, slot_set, agent_params)\n",
    "# agt = 9\n",
    "agt = 10\n",
    "agent_params['batch_size']  = batch_size\n",
    "if agt == 9:\n",
    "    agent = AgentDQN(movie_kb, act_set, slot_set, agent_params)\n",
    "else:\n",
    "    agent = DQNAgentTF(movie_kb, act_set, slot_set, agent_params)\n",
    "\n",
    "agent.set_nlg_model(nlg_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dialog Manager\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dlg_manager = DlgManager(agent, user, act_set, slot_set, movie_kb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running Episodes\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#\n",
    "status = {'successes': 0, 'count': 0, 'cumulative_reward': 0}\n",
    "# the size of validation set\n",
    "simulation_epoch_size = 100\n",
    "# the number of epochs for warm start \n",
    "warm_start_epochs = 40\n",
    "# num_episodes = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\" Warm_Start Simulation (by Rule Policy) \"\"\"\n",
    "def warm_start_simulation():\n",
    "    successes = 0\n",
    "    cumulative_reward = 0\n",
    "    cumulative_turns = 0\n",
    "    \n",
    "    res = {}\n",
    "    for episode in range(warm_start_epochs):\n",
    "        dlg_manager.init_episode()\n",
    "        episode_over = False\n",
    "        while(not episode_over):\n",
    "            episode_over, reward = dlg_manager.step()\n",
    "            cumulative_reward += reward\n",
    "            if episode_over:\n",
    "                if reward > 0: \n",
    "                    successes += 1\n",
    "#                     print (\"warm_start simulation episode %s: Success\" % (episode))\n",
    "#                 else: print (\"warm_start simulation episode %s: Fail\" % (episode))\n",
    "                cumulative_turns += dlg_manager.state_tracker.turn_count\n",
    "        \n",
    "        if len(agent.experience_replay_pool) >= agent.experience_replay_pool_size:\n",
    "            break\n",
    "    \n",
    "    agent.warm_start = 2\n",
    "    res['success_rate'] = float(successes)/simulation_epoch_size\n",
    "    res['ave_reward'] = float(cumulative_reward)/simulation_epoch_size\n",
    "    res['ave_turns'] = float(cumulative_turns)/simulation_epoch_size\n",
    "    print (\"Warm_Start %s epochs, success rate %s, ave reward %s, ave turns %s\" % (episode+1, res['success_rate'], res['ave_reward'], res['ave_turns']))\n",
    "    print (\"Current experience replay buffer size %s\" % (len(agent.experience_replay_pool)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def simulation_epoch(simulation_epoch_size):\n",
    "    successes = 0\n",
    "    cumulative_reward = 0\n",
    "    cumulative_turns = 0\n",
    "    \n",
    "    res = {}\n",
    "    for episode in range(simulation_epoch_size):\n",
    "        dlg_manager.init_episode()\n",
    "        episode_over = False\n",
    "        while(not episode_over):\n",
    "            episode_over, reward = dlg_manager.step()\n",
    "            cumulative_reward += reward\n",
    "            if episode_over:\n",
    "                if reward > 0: \n",
    "                    successes += 1\n",
    "#                     print (\"simulation episode %s: Success\" % (episode))\n",
    "#                 else: print (\"simulation episode %s: Fail\" % (episode))\n",
    "                cumulative_turns += dlg_manager.state_tracker.turn_count\n",
    "    \n",
    "    res['success_rate'] = float(successes)/simulation_epoch_size\n",
    "    res['ave_reward'] = float(cumulative_reward)/simulation_epoch_size\n",
    "    res['ave_turns'] = float(cumulative_turns)/simulation_epoch_size\n",
    "    print(\"simulation success rate %s, ave reward %s, ave turns %s\" % (res['success_rate'], res['ave_reward'], res['ave_turns']))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_episodes(count, status):\n",
    "    successes = 0\n",
    "    cumulative_reward = 0\n",
    "    cumulative_turns = 0\n",
    "    \n",
    "    \n",
    "    if agt >= 9 and params['trained_model_path'] == None and agent.warm_start == 1:\n",
    "        print ('warm_start starting ...')\n",
    "        warm_start_simulation()\n",
    "        print ('warm_start finished, start RL training ...')\n",
    "    \n",
    "    for episode in range(count):\n",
    "        print (\"----------------- Episode: %s ----------------- \" % (episode))\n",
    "        dlg_manager.init_episode()\n",
    "        episode_over = False\n",
    "        \n",
    "        while(not episode_over):\n",
    "            episode_over, reward = dlg_manager.step()\n",
    "            cumulative_reward += reward\n",
    "                \n",
    "            if episode_over:\n",
    "                if reward > 0:\n",
    "                    print (\"Successful Dialog!\")\n",
    "                    successes += 1\n",
    "#                 else: print (\"Failed Dialog!\")\n",
    "                \n",
    "                cumulative_turns += dlg_manager.state_tracker.turn_count\n",
    "        \n",
    "        # simulation\n",
    "        if agt >= 9 and params['trained_model_path'] == None:\n",
    "            agent.predict_mode = True\n",
    "            simulation_res = simulation_epoch(simulation_epoch_size)\n",
    "            \n",
    "            performance_records['success_rate'][episode] = simulation_res['success_rate']\n",
    "            performance_records['ave_turns'][episode] = simulation_res['ave_turns']\n",
    "            performance_records['ave_reward'][episode] = simulation_res['ave_reward']\n",
    "            \n",
    "            if simulation_res['success_rate'] >= best_res['success_rate']:\n",
    "                if simulation_res['success_rate'] >= success_rate_threshold: # threshold = 0.30\n",
    "                    agent.experience_replay_pool = [] \n",
    "                    simulation_epoch(simulation_epoch_size)\n",
    "                \n",
    "#             if simulation_res['success_rate'] > best_res['success_rate']:\n",
    "#                 best_model['model'] = copy.deepcopy(agent)\n",
    "#                 best_res['success_rate'] = simulation_res['success_rate']\n",
    "#                 best_res['ave_reward'] = simulation_res['ave_reward']\n",
    "#                 best_res['ave_turns'] = simulation_res['ave_turns']\n",
    "#                 best_res['epoch'] = episode\n",
    "                \n",
    "            loss = agent.train(batch_size, 1)\n",
    "            if agt == 10: \n",
    "                agent.model.update_target_params()\n",
    "            else: \n",
    "                agent.clone_dqn = copy.deepcopy(agent.dqn)\n",
    "                \n",
    "            agent.predict_mode = False\n",
    "            \n",
    "            print (\"Simulation success rate %s, Ave reward %s, Ave turns %s, Best success rate %s\" % (performance_records['success_rate'][episode], performance_records['ave_reward'][episode], performance_records['ave_turns'][episode], best_res['success_rate']))\n",
    "#             if episode % save_check_point == 0 and params['trained_model_path'] == None: # save the model every 10 episodes\n",
    "#                 save_model(params['write_model_dir'], agt, best_res['success_rate'], best_model['model'], best_res['epoch'], episode)\n",
    "#                 save_performance_records(params['write_model_dir'], agt, performance_records)\n",
    "        curve.append(successes/(episode+1))\n",
    "        losses.append(loss)\n",
    "        print(\"Progress: %s / %s, Success rate: %s / %s Avg reward: %.2f Avg turns: %.2f\" % (episode+1, count, successes, episode+1, float(cumulative_reward)/(episode+1), float(cumulative_turns)/(episode+1)))\n",
    "    print(\"Success rate: %s / %s Avg reward: %.2f Avg turns: %.2f\" % (successes, count, float(cumulative_reward)/count, float(cumulative_turns)/count))\n",
    "    status['successes'] += successes\n",
    "    status['count'] += count\n",
    "    \n",
    "#     if agt == 9 and params['traained_model_path'] == None:\n",
    "#         save_model(params['write_model_dir'], agt, float(successes)/count, best_model['model'], best_res['epoch'], count)\n",
    "#         save_performance_records(params['write_model_dir'], agt, performance_records)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run a Warm Start"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train & Eval\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "warm_start starting ...\n",
      "Warm_Start 40 epochs, success rate 0.13, ave reward -3.2, ave turns 6.4\n",
      "Current experience replay buffer size 320\n",
      "warm_start finished, start RL training ...\n",
      "----------------- Episode: 0 ----------------- \n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Cannot feed value of shape (1, 377) for Tensor 'Placeholder:0', which has shape '(?, 333)'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-c7d112675a73>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mlosses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarm_start\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mrun_episodes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-18-9270d26611ac>\u001b[0m in \u001b[0;36mrun_episodes\u001b[0;34m(count, status)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0;32mwhile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mnot\u001b[0m \u001b[0mepisode_over\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m             \u001b[0mepisode_over\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdlg_manager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m             \u001b[0mcumulative_reward\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/dengl11/CS234-Project/dlg_manager/dlg_manager.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, record_training_data)\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0;31m########################################################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_tracker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_state_for_agent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 82\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magt_action\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_to_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m         \u001b[0;31m########################################################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/dengl11/CS234-Project/agent/dqn_agt.py\u001b[0m in \u001b[0;36mstate_to_action\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrepresentation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_state_representation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrepresentation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0mact_slot_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeasible_actions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'act_slot_response'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mact_slot_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'act_slot_value_response'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/dengl11/CS234-Project/agent/dqn_agt_tf.py\u001b[0m in \u001b[0;36mrun_policy\u001b[0;34m(self, representation)\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m                 \u001b[0;31m# return self.dqn.predict(representation, {}, predict_model=True)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrepresentation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/dengl11/CS234-Project/alg/dqn_tf.py\u001b[0m in \u001b[0;36mpredict_action\u001b[0;34m(self, states)\u001b[0m\n\u001b[1;32m    193\u001b[0m             \u001b[0mactions\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    194\u001b[0m         \"\"\"\n\u001b[0;32m--> 195\u001b[0;31m         \u001b[0maction_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstates\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstates\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    196\u001b[0m         \u001b[0;31m# print(\"action_values=\", action_values)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0mactions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction_values\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    776\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    777\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 778\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    779\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    780\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    959\u001b[0m                 \u001b[0;34m'Cannot feed value of shape %r for Tensor %r, '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    960\u001b[0m                 \u001b[0;34m'which has shape %r'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 961\u001b[0;31m                 % (np_val.shape, subfeed_t.name, str(subfeed_t.get_shape())))\n\u001b[0m\u001b[1;32m    962\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_feedable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubfeed_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    963\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Tensor %s may not be fed.'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0msubfeed_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Cannot feed value of shape (1, 377) for Tensor 'Placeholder:0', which has shape '(?, 333)'"
     ]
    }
   ],
   "source": [
    "performance_records = {}\n",
    "performance_records['success_rate'] = {}\n",
    "performance_records['ave_turns'] = {}\n",
    "performance_records['ave_reward'] = {}\n",
    "\n",
    "best_model = {}\n",
    "best_res = {'success_rate': 0, 'ave_reward':float('-inf'), 'ave_turns': float('inf'), 'epoch':0}\n",
    "\n",
    "curve = []\n",
    "losses = []\n",
    "agent.warm_start = 1\n",
    "run_episodes(100, status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_learning_curve(curve)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_loss_curve(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_loss_curve(losses[-100:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# saver.save(agent.model.sess, \"trained_model/tf_400/\", global_step = 400)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver.save(agent.model.sess, \"trained_model/movie_100/model.ckpt\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "492/600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}