DQN-tensorflow-Flights-Trained.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DQN\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training of DQN in Tensorflow\n",
    "--------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "code_folding": [],
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "#\n",
    "from util import *\n",
    "from dlg_manager import *\n",
    "from alg import *\n",
    "from agent import *\n",
    "from user_sim import *\n",
    "from state_tracker import *\n",
    "import random\n",
    "from config import *\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from nlg import *\n",
    "from six.moves import cPickle as pickle\n",
    "import IPython\n",
    "import copy, argparse, json\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Action Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "code_folding": [],
    "init_cell": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keys = 11\n",
      "Sample of dict:\n",
      "- not_sure: 10\n",
      "- thanks: 7\n",
      "- multiple_choice: 6\n",
      "- greeting: 4\n",
      "- confirm_answer: 3\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "act_set_path = './data/dia_acts.txt'\n",
    "act_set = text_to_dict(act_set_path)\n",
    "sample_dict(act_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### slot set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": [],
    "init_cell": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keys = 7\n",
      "Sample of dict:\n",
      "- flightDate1: 2\n",
      "- origin1: 3\n",
      "- ticket: 5\n",
      "- taskcomplete: 6\n",
      "- travelers: 4\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "slots_set_path = \"./parser/slot_set.txt\"\n",
    "slot_set = text_to_dict(slots_set_path)\n",
    "sample_dict(slot_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### flight dic: info about flights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": [],
    "init_cell": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keys = 1000\n",
      "Sample of dict:\n",
      "- 615: {'destination1': 'MCO', 'flightDate2': '3', 'travelers': '3', 'origin1': 'YYZ', 'flightDate1': '0'}\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "flight_kb_path = \"./parser/fkb.json.p\"\n",
    "flight_kb = pickle.load(open(flight_kb_path, 'rb'), encoding=\"latin\")\n",
    "sample_dict(flight_kb, sample_size=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Language Generator (pretrained)\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#\n",
    "nlg_model_path ='data/trained_model/nlg/lstm_tanh_relu_[1468202263.38]_2_0.610.p'\n",
    "nlg_model = Nlg()\n",
    "nlg_model.load_nlg_model(nlg_model_path)\n",
    "diaact_nl_pairs_path = \"./parser/flight.nl.pairs.json\"\n",
    "nlg_model.load_predefine_act_nl_pairs(diaact_nl_pairs_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "code_folding": [],
    "init_cell": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_path: .\\data\\movieMultiLine.Annot.Corrected.Final.v3.csv\n",
      "learning_rate: 0.001\n",
      "momentum: 0.1\n",
      "activation_func: relu\n",
      "cv_fold: 6\n",
      "smooth_eps: 1e-08\n",
      "valid_test: 0\n",
      "save_check_point: 20\n",
      "batch_size: 16\n",
      "model: lstm_tanh\n",
      "hidden_size: 100\n",
      "init_rnn: 0\n",
      "dia_slot_val: 2\n",
      "slot_set: data/slot_set.txt\n",
      "write_model_dir: .\\checkpoints\\template\\07102016\\\n",
      "sdgtype: rmsprop\n",
      "eva_metric: 2\n",
      "feed_recurrence: 0\n",
      "pretrained_model_path: None\n",
      "act_set: data/dia_acts.txt\n",
      "check_point: 20\n",
      "decay_rate: 0.999\n",
      "slot_rep: 1\n",
      "max_epochs: 200\n",
      "grad_clip: -0.0001\n",
      "trained_model_path: None\n",
      "split_method: 1\n",
      "reg_cost: 0.001\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "model_params = pickle.load(open(nlg_model_path, 'rb'), encoding='latin1')\n",
    "params = model_params['params']\n",
    "params['batch_size'] = 16\n",
    "batch_size = 16\n",
    "save_check_point = 20\n",
    "params['trained_model_path'] = None\n",
    "for k in params:\n",
    "    print(\"{}: {}\".format(k, params[k]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# User Simulator\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### goal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "code_folding": [],
    "init_cell": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "goals length: 128\n",
      "Sample the first goal: \n",
      "{'inform_slots': {'destination1': 'MCO', 'flightDate2': '3', 'travelers': '4', 'origin1': 'MUC', 'flightDate1': '2'}, 'request_slots': {}, 'diaact': 'request'}\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "goal_file_path = './parser/fg.json.p'\n",
    "all_goal_set = pickle.load(open(goal_file_path, 'rb'), encoding=\"latin\")\n",
    "print(\"goals length: {}\".format(len(all_goal_set)))\n",
    "print(\"Sample the first goal: \\n{}\".format(all_goal_set[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split goal set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "102\n",
      "0\n",
      "26\n",
      "128\n"
     ]
    }
   ],
   "source": [
    "# split goal set\n",
    "split_fold = params.get('split_fold', 5)\n",
    "goal_set = {'train':[], 'valid':[], 'test':[], 'all':[]}\n",
    "for u_goal_id, u_goal in enumerate(all_goal_set):\n",
    "    if u_goal_id % split_fold == 1: goal_set['test'].append(u_goal)\n",
    "    else: goal_set['train'].append(u_goal)\n",
    "    goal_set['all'].append(u_goal)\n",
    "print(len(goal_set['train']))\n",
    "print(len(goal_set['valid']))\n",
    "print(len(goal_set['test']))\n",
    "print(len(goal_set['all']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### user simulator param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#\n",
    "usersim_params = {}\n",
    "usersim_params['max_turn'] = 40\n",
    "usersim_params['slot_err_prob'] = 0.00\n",
    "# slot_err_mode: 0 for slot_val only; 1 for three errs\n",
    "usersim_params['slot_err_mode'] = 0\n",
    "usersim_params['intent_err_prob'] = 0\n",
    "# run_mode: 0 for default NL; 1 for dia_act; 2 for both\n",
    "usersim_params['run_mode'] = 0\n",
    "# 0 for dia_act level; 1 for NL level\n",
    "usersim_params['act_level'] = 0\n",
    "# train/test/all; default is all\n",
    "usersim_params['learn_phase'] = 'all'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a flights dictionary for user simulator - slot:possible values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keys = 20\n",
      "Sample of dict:\n",
      "- theater_chain: ['regal meridian', 'amc', 'amc loews stony brook 17', 'amc hamilton 24', 'amc pacific place 11', 'regency', 'amc loews waterfront 22', 'century', 'amc star john r 15', 'amc star southfield', 'amc lowes oak tree 6', 'amc loews oak tree 6', 'amc showplace carbondale 8', 'century eastport 16', 'amc theater', 'amc ahwatukee 24', ' amc mesa grand 24', 'amc showplace carbondale']\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "flight_dict_path = './parser/dicts.v3.p'\n",
    "flight_dictionary = pickle.load(open(flight_dict_path, 'rb'), encoding=\"latin\")\n",
    "samples = sample_dict(flight_dictionary, sample_size=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Create a User"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "user = RuleSimulator(flight_dictionary, act_set, slot_set, goal_set, usersim_params)\n",
    "# user = AlternateSimulator(flight_dictionary, act_set, slot_set, goal_set, usersim_params)\n",
    "user.set_nlg_model(nlg_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agent\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pretrained model path = None\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "agent_params = {}\n",
    "# maximum length of each dialog (default=20, 0=no maximum length)\n",
    "agent_params['max_turn'] = 40\n",
    "# Epsilon to determine stochasticity of epsilon-greedy agent policies\n",
    "agent_params['epsilon'] = 0\n",
    "# run_mode: 0 for default NL; 1 for dia_act; 2 for both\n",
    "agent_params['agent_run_mode'] = 3\n",
    "# 0 for dia_act level; 1 for NL level\n",
    "agent_params['agent_act_level'] = 0\n",
    "\n",
    "############### DQN #################\n",
    "# the size for experience replay\n",
    "agent_params['experience_replay_pool_size'] = 10000\n",
    "# # the hidden size for DQN\n",
    "agent_params['dqn_hidden_size'] = 60\n",
    "agent_params['batch_size'] = 16\n",
    "# # gamma for DQN\n",
    "agent_params['gamma'] = 0.9\n",
    "# # predict model for DQN\n",
    "agent_params['predict_mode'] = True\n",
    "agent_params['trained_model_path'] = params['pretrained_model_path']\n",
    "#####################################\n",
    "print(\"pretrained model path = {}\".format(agent_params['trained_model_path']))\n",
    "# 0: no warm start; 1: warm start for training\n",
    "agent_params['warm_start'] = 1\n",
    "# run_mode: 0 for NL; 1 for dia_act\n",
    "agent_params['cmd_input_mode'] = 0\n",
    "\n",
    "success_rate_threshold = 0.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create an agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "code_folding": [],
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80\n",
      "INFO:tensorflow:Restoring parameters from trained_model/flight_100/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "# agent = RequestBasicsAgent(movie_kb, act_set, slot_set, agent_params)\n",
    "# agent = AgentDQN(movie_kb, act_set, slot_set, agent_params)\n",
    "# agt = 9\n",
    "agt = 10\n",
    "agent_params['batch_size']  = batch_size\n",
    "if agt == 9:\n",
    "    agent = AgentDQN(flight_kb, act_set, slot_set, agent_params)\n",
    "else:\n",
    "    agent = DQNAgentTF(flight_kb, act_set, slot_set, agent_params, transfer=True, path=\"trained_model/flight_100/model.ckpt\")\n",
    "\n",
    "agent.set_nlg_model(nlg_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dialog Manager\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dlg_manager = DlgManager(agent, user, act_set, slot_set, flight_kb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running Episodes\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [],
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#\n",
    "status = {'successes': 0, 'count': 0, 'cumulative_reward': 0}\n",
    "# the size of validation set\n",
    "simulation_epoch_size = 100\n",
    "# the number of epochs for warm start \n",
    "warm_start_epochs = 100\n",
    "# num_episodes = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\" Warm_Start Simulation (by Rule Policy) \"\"\"\n",
    "def warm_start_simulation():\n",
    "    successes = 0\n",
    "    cumulative_reward = 0\n",
    "    cumulative_turns = 0\n",
    "    \n",
    "    res = {}\n",
    "    for episode in range(warm_start_epochs):\n",
    "        dlg_manager.init_episode()\n",
    "        episode_over = False\n",
    "        while(not episode_over):\n",
    "            episode_over, reward = dlg_manager.step()\n",
    "            cumulative_reward += reward\n",
    "            if episode_over:\n",
    "                if reward > 0: \n",
    "                    successes += 1\n",
    "#                     print (\"warm_start simulation episode %s: Success\" % (episode))\n",
    "#                 else: print (\"warm_start simulation episode %s: Fail\" % (episode))\n",
    "                cumulative_turns += dlg_manager.state_tracker.turn_count\n",
    "        \n",
    "        if len(agent.experience_replay_pool) >= agent.experience_replay_pool_size:\n",
    "            break\n",
    "    \n",
    "    agent.warm_start = 2\n",
    "    res['success_rate'] = float(successes)/simulation_epoch_size\n",
    "    res['ave_reward'] = float(cumulative_reward)/simulation_epoch_size\n",
    "    res['ave_turns'] = float(cumulative_turns)/simulation_epoch_size\n",
    "    print (\"Warm_Start %s epochs, success rate %s, ave reward %s, ave turns %s\" % (episode+1, res['success_rate'], res['ave_reward'], res['ave_turns']))\n",
    "    print (\"Current experience replay buffer size %s\" % (len(agent.experience_replay_pool)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def simulation_epoch(simulation_epoch_size):\n",
    "    successes = 0\n",
    "    cumulative_reward = 0\n",
    "    cumulative_turns = 0\n",
    "    \n",
    "    res = {}\n",
    "    for episode in range(simulation_epoch_size):\n",
    "        dlg_manager.init_episode()\n",
    "        episode_over = False\n",
    "        while(not episode_over):\n",
    "            episode_over, reward = dlg_manager.step()\n",
    "            cumulative_reward += reward\n",
    "            if episode_over:\n",
    "                if reward > 0: \n",
    "                    successes += 1\n",
    "#                     print (\"simulation episode %s: Success\" % (episode))\n",
    "#                 else: print (\"simulation episode %s: Fail\" % (episode))\n",
    "                cumulative_turns += dlg_manager.state_tracker.turn_count\n",
    "    \n",
    "    res['success_rate'] = float(successes)/simulation_epoch_size\n",
    "    res['ave_reward'] = float(cumulative_reward)/simulation_epoch_size\n",
    "    res['ave_turns'] = float(cumulative_turns)/simulation_epoch_size\n",
    "    print(\"simulation success rate %s, ave reward %s, ave turns %s\" % (res['success_rate'], res['ave_reward'], res['ave_turns']))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [],
    "collapsed": true,
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "def run_episodes(count, status):\n",
    "    successes = 0\n",
    "    cumulative_reward = 0\n",
    "    cumulative_turns = 0\n",
    "    \n",
    "    \n",
    "    if agt >= 9 and params['trained_model_path'] == None and agent.warm_start == 1:\n",
    "        print ('warm_start starting ...')\n",
    "        warm_start_simulation()\n",
    "        print ('warm_start finished, start RL training ...')\n",
    "    \n",
    "    for episode in range(count):\n",
    "        print (\"----------------- Episode: %s ----------------- \" % (episode))\n",
    "        dlg_manager.init_episode()\n",
    "        episode_over = False\n",
    "        \n",
    "        while(not episode_over):\n",
    "            episode_over, reward = dlg_manager.step()\n",
    "            cumulative_reward += reward\n",
    "                \n",
    "            if episode_over:\n",
    "                if reward > 0:\n",
    "                    print (\"Successful Dialog!\")\n",
    "                    successes += 1\n",
    "#                 else: print (\"Failed Dialog!\")\n",
    "                \n",
    "                cumulative_turns += dlg_manager.state_tracker.turn_count\n",
    "        \n",
    "        # simulation\n",
    "        if agt >= 9 and params['trained_model_path'] == None:\n",
    "            agent.predict_mode = True\n",
    "            simulation_res = simulation_epoch(simulation_epoch_size)\n",
    "            \n",
    "            performance_records['success_rate'][episode] = simulation_res['success_rate']\n",
    "            performance_records['ave_turns'][episode] = simulation_res['ave_turns']\n",
    "            performance_records['ave_reward'][episode] = simulation_res['ave_reward']\n",
    "            \n",
    "            if simulation_res['success_rate'] >= best_res['success_rate']:\n",
    "                if simulation_res['success_rate'] >= success_rate_threshold: # threshold = 0.30\n",
    "                    agent.experience_replay_pool = [] \n",
    "                    simulation_epoch(simulation_epoch_size)\n",
    "                \n",
    "#             if simulation_res['success_rate'] > best_res['success_rate']:\n",
    "#                 best_model['model'] = copy.deepcopy(agent)\n",
    "#                 best_res['success_rate'] = simulation_res['success_rate']\n",
    "#                 best_res['ave_reward'] = simulation_res['ave_reward']\n",
    "#                 best_res['ave_turns'] = simulation_res['ave_turns']\n",
    "#                 best_res['epoch'] = episode\n",
    "                \n",
    "            loss = agent.train(batch_size, 1)\n",
    "            if agt == 10: \n",
    "                agent.model.update_target_params()\n",
    "            else: \n",
    "                agent.clone_dqn = copy.deepcopy(agent.dqn)\n",
    "                \n",
    "            agent.predict_mode = False\n",
    "            \n",
    "            print (\"Simulation success rate %s, Ave reward %s, Ave turns %s, Best success rate %s\" % (performance_records['success_rate'][episode], performance_records['ave_reward'][episode], performance_records['ave_turns'][episode], best_res['success_rate']))\n",
    "#             if episode % save_check_point == 0 and params['trained_model_path'] == None: # save the model every 10 episodes\n",
    "#                 save_model(params['write_model_dir'], agt, best_res['success_rate'], best_model['model'], best_res['epoch'], episode)\n",
    "#                 save_performance_records(params['write_model_dir'], agt, performance_records)\n",
    "        curve.append(successes/(episode+1))\n",
    "        losses.append(loss)\n",
    "        print(\"Progress: %s / %s, Success rate: %s / %s Avg reward: %.2f Avg turns: %.2f\" % (episode+1, count, successes, episode+1, float(cumulative_reward)/(episode+1), float(cumulative_turns)/(episode+1)))\n",
    "    print(\"Success rate: %s / %s Avg reward: %.2f Avg turns: %.2f\" % (successes, count, float(cumulative_reward)/count, float(cumulative_turns)/count))\n",
    "    status['successes'] += successes\n",
    "    status['count'] += count\n",
    "    \n",
    "#     if agt == 9 and params['traained_model_path'] == None:\n",
    "#         save_model(params['write_model_dir'], agt, float(successes)/count, best_model['model'], best_res['epoch'], count)\n",
    "#         save_performance_records(params['write_model_dir'], agt, performance_records)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run a Warm Start"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train & Eval\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "performance_records = {}\n",
    "performance_records['success_rate'] = {}\n",
    "performance_records['ave_turns'] = {}\n",
    "performance_records['ave_reward'] = {}\n",
    "\n",
    "best_model = {}\n",
    "best_res = {'success_rate': 0, 'ave_reward':float('-inf'), 'ave_turns': float('inf'), 'epoch':0}\n",
    "\n",
    "curve = []\n",
    "losses = []\n",
    "agent.warm_start = 1\n",
    "run_episodes(100, status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_learning_curve(curve)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_loss_curve(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_loss_curve(losses[-100:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver.save(agent.model.sess, \"trained_model/flight_100/model.ckpt\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Initialization Cell",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}