Seq2Seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  reading data line 100000\n",
      "  reading data line 200000\n",
      "  reading data line 300000\n",
      "  reading data line 400000\n",
      "  reading data line 500000\n",
      "  reading data line 600000\n",
      "  reading data line 700000\n",
      "  reading data line 800000\n",
      "  reading data line 900000\n",
      "  reading data line 1000000\n",
      "  reading data line 1100000\n",
      "  reading data line 1200000\n",
      "  reading data line 1300000\n",
      "  reading data line 1400000\n",
      "  reading data line 1500000\n",
      "  reading data line 1600000\n",
      "  reading data line 1700000\n",
      "  reading data line 1800000\n",
      "  reading data line 1900000\n",
      "  reading data line 2000000\n",
      "  reading data line 2100000\n",
      "  reading data line 2200000\n",
      "  reading data line 2300000\n",
      "  reading data line 2400000\n",
      "  reading data line 2500000\n",
      "  reading data line 2600000\n",
      "  reading data line 2700000\n",
      "  reading data line 2800000\n",
      "  reading data line 2900000\n",
      "  reading data line 3000000\n",
      "  reading data line 3100000\n",
      "  reading data line 3200000\n",
      "  reading data line 3300000\n",
      "  reading data line 3400000\n",
      "  reading data line 3500000\n",
      "  reading data line 3600000\n",
      "  reading data line 3700000\n",
      "  reading data line 3800000\n",
      "  reading data line 3900000\n",
      "  reading data line 4000000\n",
      "  reading data line 4100000\n",
      "  reading data line 4200000\n",
      "  reading data line 4300000\n",
      "  reading data line 4400000\n",
      "  reading data line 4500000\n",
      "  reading data line 4600000\n",
      "  reading data line 4700000\n",
      "  reading data line 4800000\n",
      "  reading data line 4900000\n",
      "  reading data line 5000000\n",
      "  reading data line 5100000\n",
      "  reading data line 5200000\n",
      "  reading data line 5300000\n",
      "  reading data line 5400000\n",
      "  reading data line 5500000\n",
      "  reading data line 5600000\n",
      "  reading data line 5700000\n",
      "  reading data line 5800000\n",
      "  reading data line 5900000\n",
      "  reading data line 6000000\n",
      "  reading data line 6100000\n",
      "  reading data line 6200000\n",
      "  reading data line 6300000\n",
      "  reading data line 6400000\n",
      "  reading data line 6500000\n",
      "  reading data line 6600000\n",
      "  reading data line 6700000\n",
      "  reading data line 6800000\n",
      "  reading data line 6900000\n",
      "  reading data line 7000000\n",
      "  reading data line 7100000\n",
      "  reading data line 7200000\n",
      "  reading data line 7300000\n",
      "  reading data line 7400000\n",
      "  reading data line 7500000\n",
      "  reading data line 7600000\n",
      "  reading data line 7700000\n",
      "  reading data line 7800000\n",
      "  reading data line 7900000\n",
      "  reading data line 8000000\n",
      "  reading data line 8100000\n",
      "  reading data line 8200000\n",
      "  reading data line 8300000\n",
      "  reading data line 8400000\n",
      "  reading data line 8500000\n",
      "  reading data line 8600000\n",
      "  reading data line 8700000\n",
      "  reading data line 8800000\n",
      "  reading data line 8900000\n",
      "  reading data line 9000000\n",
      "  reading data line 9100000\n",
      "  reading data line 9200000\n",
      "  reading data line 9300000\n",
      "  reading data line 9400000\n",
      "  reading data line 9500000\n",
      "  reading data line 9600000\n",
      "  reading data line 9700000\n",
      "  reading data line 9800000\n",
      "  reading data line 9900000\n",
      "  reading data line 10000000\n",
      "  reading data line 10100000\n",
      "  reading data line 10200000\n",
      "  reading data line 10300000\n",
      "  reading data line 10400000\n",
      "  reading data line 10500000\n",
      "  reading data line 10600000\n",
      "  reading data line 10700000\n",
      "  reading data line 10800000\n",
      "  reading data line 10900000\n",
      "  reading data line 11000000\n",
      "  reading data line 11100000\n",
      "  reading data line 11200000\n",
      "  reading data line 11300000\n",
      "  reading data line 11400000\n",
      "  reading data line 11500000\n",
      "  reading data line 11600000\n",
      "  reading data line 11700000\n",
      "  reading data line 11800000\n",
      "  reading data line 11900000\n",
      "  reading data line 12000000\n",
      "  reading data line 12100000\n",
      "  reading data line 12200000\n",
      "  reading data line 12300000\n",
      "  reading data line 12400000\n",
      "  reading data line 12500000\n",
      "  reading data line 12600000\n",
      "  reading data line 12700000\n",
      "  reading data line 12800000\n",
      "  reading data line 12900000\n",
      "  reading data line 13000000\n",
      "  reading data line 13100000\n",
      "  reading data line 13200000\n",
      "  reading data line 13300000\n",
      "  reading data line 13400000\n",
      "  reading data line 13500000\n",
      "  reading data line 13600000\n",
      "  reading data line 13700000\n",
      "  reading data line 13800000\n",
      "  reading data line 13900000\n",
      "  reading data line 14000000\n",
      "  reading data line 14100000\n",
      "  reading data line 14200000\n",
      "  reading data line 14300000\n",
      "  reading data line 14400000\n",
      "  reading data line 14500000\n",
      "  reading data line 14600000\n",
      "  reading data line 14700000\n",
      "  reading data line 14800000\n",
      "  reading data line 14900000\n",
      "  reading data line 15000000\n",
      "  reading data line 15100000\n",
      "  reading data line 15200000\n",
      "  reading data line 15300000\n",
      "  reading data line 15400000\n",
      "  reading data line 15500000\n",
      "  reading data line 15600000\n",
      "  reading data line 15700000\n",
      "  reading data line 15800000\n",
      "  reading data line 15900000\n",
      "  reading data line 16000000\n",
      "  reading data line 16100000\n",
      "  reading data line 16200000\n",
      "  reading data line 16300000\n",
      "  reading data line 16400000\n",
      "  reading data line 16500000\n",
      "  reading data line 16600000\n",
      "  reading data line 16700000\n",
      "  reading data line 16800000\n",
      "  reading data line 16900000\n",
      "  reading data line 17000000\n",
      "  reading data line 17100000\n",
      "  reading data line 17200000\n",
      "  reading data line 17300000\n",
      "  reading data line 17400000\n",
      "  reading data line 17500000\n",
      "  reading data line 17600000\n",
      "  reading data line 17700000\n",
      "  reading data line 17800000\n",
      "  reading data line 17900000\n",
      "  reading data line 18000000\n",
      "  reading data line 18100000\n",
      "  reading data line 18200000\n",
      "  reading data line 18300000\n",
      "  reading data line 18400000\n",
      "  reading data line 18500000\n",
      "  reading data line 18600000\n",
      "  reading data line 18700000\n",
      "  reading data line 18800000\n",
      "  reading data line 18900000\n",
      "  reading data line 19000000\n",
      "  reading data line 19100000\n",
      "  reading data line 19200000\n",
      "  reading data line 19300000\n",
      "  reading data line 19400000\n",
      "  reading data line 19500000\n",
      "  reading data line 19600000\n",
      "  reading data line 19700000\n",
      "  reading data line 19800000\n",
      "  reading data line 19900000\n",
      "  reading data line 20000000\n",
      "  reading data line 20100000\n",
      "  reading data line 20200000\n",
      "  reading data line 20300000\n",
      "  reading data line 20400000\n",
      "  reading data line 20500000\n",
      "  reading data line 20600000\n",
      "  reading data line 20700000\n",
      "  reading data line 20800000\n",
      "  reading data line 20900000\n",
      "  reading data line 21000000\n",
      "  reading data line 21100000\n",
      "  reading data line 21200000\n",
      "  reading data line 21300000\n",
      "  reading data line 21400000\n",
      "  reading data line 21500000\n",
      "  reading data line 21600000\n",
      "  reading data line 21700000\n",
      "  reading data line 21800000\n",
      "  reading data line 21900000\n",
      "  reading data line 22000000\n",
      "  reading data line 22100000\n",
      "  reading data line 22200000\n",
      "  reading data line 22300000\n",
      "  reading data line 22400000\n",
      "  reading data line 22500000\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import tensorflow as tf, re, time, math\n",
    "from wmt_data import *\n",
    "from seq2seq_model import *\n",
    "from tensorflow.python.framework import ops\n",
    "from compute_bleu import *\n",
    "\n",
    "buckets = [(5, 10), (10, 15), (20, 25), (40, 50)]\n",
    "V_en = 50000; V_fr = 70000; max_train_data_size = 0\n",
    "learning_rate = 0.5; num_samples = 512\n",
    "state_size = 512; num_layers = 3\n",
    "\n",
    "en_train, fr_train, en_dev, fr_dev, en_vocab_path, fr_vocab_path = prepare_wmt_data(V_en, V_fr)\n",
    "dev_set = read_data(en_dev, fr_dev, buckets)\n",
    "train_set = read_data(en_train, fr_train, buckets, max_train_data_size)\n",
    "\n",
    "def create_model():\n",
    "    ops.reset_default_graph()\n",
    "    return Seq2SeqModel(V_en, V_fr, buckets, state_size, num_layers, 5.0, learning_rate, num_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model\n",
      "Variable:0 ()\n",
      "Variable_1:0 ()\n",
      "proj_w:0 (70000, 512)\n",
      "proj_b:0 (70000,)\n",
      "seq2seq/encoding/embedding_en:0 (50000, 512)\n",
      "seq2seq/encoding/RNN/BasicLSTMCell/Linear/Matrix:0 (1024, 2048)\n",
      "seq2seq/encoding/RNN/BasicLSTMCell/Linear/Bias:0 (2048,)\n",
      "seq2seq/decoder/embedding_de:0 (70000, 512)\n",
      "seq2seq/decoder/w_a:0 (512, 512)\n",
      "seq2seq/decoder/w_c:0 (1024, 512)\n",
      "seq2seq/decoder/BasicLSTMCell/Linear/Matrix:0 (1024, 2048)\n",
      "seq2seq/decoder/BasicLSTMCell/Linear/Bias:0 (2048,)\n"
     ]
    }
   ],
   "source": [
    "model = create_model()\n",
    "print \"Loaded model\"\n",
    "\n",
    "for v in tf.all_variables():\n",
    "    print v.name, v.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "t = time.time()\n",
    "# model = create_model()\n",
    "print \"Loaded model\", (time.time() - t)\n",
    "saver = tf.train.Saver(tf.all_variables(), max_to_keep=2)\n",
    "\n",
    "vocab_en, rev_vocab_en = initialize_vocabulary(en_vocab_path)\n",
    "vocab_fr, rev_vocab_fr = initialize_vocabulary(fr_vocab_path)\n",
    "\n",
    "def train(name, sampled=True, steps=50000):\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.initialize_all_variables())\n",
    "        saver.restore(sess, \"wmt/my_full_attns.ckpt-300000\")\n",
    "        steps_per_checkpoint = 250; batch_size = 128\n",
    "\n",
    "        train_bucket_sizes = [len(train_set[b]) for b in xrange(len(buckets))]\n",
    "        train_total_size = float(sum(train_bucket_sizes))\n",
    "\n",
    "        train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes))]\n",
    "\n",
    "        # This is the training loop.\n",
    "        step_time, loss = 0.0, 0.0; global_step = 0;\n",
    "        previous_losses = []\n",
    "        for current_step in range(steps):\n",
    "\n",
    "            random_number_01 = np.random.random_sample() # Smart\n",
    "            bucket_id = min([i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01])\n",
    "\n",
    "            start_time = time.time()\n",
    "            encoder_inputs, decoder_inputs, target_weights = get_batch(train_set, batch_size, buckets, bucket_id)\n",
    "            step_loss, _ = model.train_step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, sampled)\n",
    "            step_time += (time.time() - start_time) / steps_per_checkpoint\n",
    "            loss += step_loss / steps_per_checkpoint\n",
    "            if current_step % steps_per_checkpoint == (-1 % steps_per_checkpoint):\n",
    "                perplexity = math.exp(float(loss)) if loss < 300 else float(\"inf\")\n",
    "                global_step = model.global_step.eval()\n",
    "                if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):\n",
    "                    sess.run(model.learning_rate_decay_op)\n",
    "                previous_losses.append(loss)\n",
    "                bleus = computeBleu(sess, model, dev_set, rev_vocab_en, rev_vocab_fr)\n",
    "                f = open('bleus_'+name+'.txt','a'); f.write(str(global_step)+' '+str(bleus)+'\\n'); f.close();\n",
    "                print (\"global step %d learning rate %.4f step-time %.2f perplexity %.2f bleu %.2f\" % (global_step, model.learning_rate.eval(), step_time, perplexity, bleus))\n",
    "                step_time, loss = 0.0, 0.0\n",
    "            if current_step % 1000 == (-1 % 1000):\n",
    "                t = time.time()\n",
    "                saver.save(sess, \"wmt/\"+name+\".ckpt\", global_step=model.global_step)\n",
    "                print \"Saving takes\", time.time()-t, \"seconds\"\n",
    "# train(\"attns\", True, 200001)\n",
    "train(\"w\", False, 300000)\n",
    "# train(\"full\", False, 23000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable:0 ()\n",
      "Variable_1:0 ()\n",
      "proj_w:0 (70000, 512)\n",
      "proj_b:0 (70000,)\n",
      "seq2seq/embedding_en:0 (50000, 512)\n",
      "seq2seq/encoding/RNN/BasicLSTMCell/Linear/Matrix:0 (1024, 2048)\n",
      "seq2seq/encoding/RNN/BasicLSTMCell/Linear/Bias:0 (2048,)\n",
      "seq2seq/decoder/embedding_de:0 (50000, 512)\n",
      "seq2seq/decoder/BasicLSTMCell/Linear/Matrix:0 (1024, 2048)\n",
      "seq2seq/decoder/BasicLSTMCell/Linear/Bias:0 (2048,)\n"
     ]
    }
   ],
   "source": [
    "for v in tf.all_variables():\n",
    "    print v.name, v.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "def readSSV(fn):\n",
    "    f = open(fn)\n",
    "    dic = {};\n",
    "    for line in iter(f):\n",
    "        attrs = line.split(' ')\n",
    "        div[int(attrs[0])] = float(attrs[1])\n",
    "    f.close()\n",
    "    return dic\n",
    "\n",
    "full_run = readSSV('sampled_bleus.txt')\n",
    "sampled_run = readSSV('sampled_bleus.txt')\n",
    "\n",
    "plt.plot(full_run.keys(), full_run.values(), label=\"Full Softmax\")\n",
    "plt.plot(sampled_run.keys(), sampled_run.values(), label=\"Sampled Softmax\")\n",
    "plt.legend(loc='upper left')\n",
    "plt.xlabel(\"Iterations\")\n",
    "plt.ylabel(\"BLEU Score\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NotFoundError",
     "evalue": "Tensor name \"embedding\" not found in checkpoint files wmt/translate.ckpt-176800\n\t [[Node: save_6/RestoreV2_2 = RestoreV2[dtypes=[DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/cpu:0\"](_recv_save_6/Const_0, save_6/RestoreV2_2/tensor_names, save_6/RestoreV2_2/shape_and_slices)]]\n\t [[Node: save_6/RestoreV2_10/_15 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device_incarnation=1, tensor_name=\"edge_160_save_6/RestoreV2_10\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n\nCaused by op u'save_6/RestoreV2_2', defined at:\n  File \"/usr/lib/python2.7/runpy.py\", line 162, in _run_module_as_main\n    \"__main__\", fname, loader, pkg_name)\n  File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code\n    exec code in run_globals\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python2.7/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/usr/local/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py\", line 162, in start\n    super(ZMQIOLoop, self).start()\n  File \"/usr/local/lib/python2.7/dist-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-12-5d8c121d9cf8>\", line 3, in <module>\n    saver = tf.train.Saver(tf.all_variables())\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 1078, in __init__\n    self.build()\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 1107, in build\n    restore_sequentially=self._restore_sequentially)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 705, in build\n    restore_sequentially, reshape)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 442, in _AddRestoreOps\n    tensors = self.restore_op(filename_tensor, saveable, preferred_shard)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 281, in restore_op\n    [spec.tensor.dtype])[0])\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_io_ops.py\", line 439, in restore_v2\n    dtypes=dtypes, name=name)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py\", line 749, in apply_op\n    op_def=op_def)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 2386, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 1298, in __init__\n    self._traceback = _extract_stack()\n\nNotFoundError (see above for traceback): Tensor name \"embedding\" not found in checkpoint files wmt/translate.ckpt-176800\n\t [[Node: save_6/RestoreV2_2 = RestoreV2[dtypes=[DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/cpu:0\"](_recv_save_6/Const_0, save_6/RestoreV2_2/tensor_names, save_6/RestoreV2_2/shape_and_slices)]]\n\t [[Node: save_6/RestoreV2_10/_15 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device_incarnation=1, tensor_name=\"edge_160_save_6/RestoreV2_10\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNotFoundError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-5d8c121d9cf8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0msaver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"wmt/translate.ckpt-176800\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"embedding_rnn_seq2seq\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"RNN\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.pyc\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, sess, save_path)\u001b[0m\n\u001b[1;32m   1435\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1436\u001b[0m     sess.run(self.saver_def.restore_op_name,\n\u001b[0;32m-> 1437\u001b[0;31m              {self.saver_def.filename_tensor_name: save_path})\n\u001b[0m\u001b[1;32m   1438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1439\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    715\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 717\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    718\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    719\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    913\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    914\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 915\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    916\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m--> 965\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m    983\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    984\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 985\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    986\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    987\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNotFoundError\u001b[0m: Tensor name \"embedding\" not found in checkpoint files wmt/translate.ckpt-176800\n\t [[Node: save_6/RestoreV2_2 = RestoreV2[dtypes=[DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/cpu:0\"](_recv_save_6/Const_0, save_6/RestoreV2_2/tensor_names, save_6/RestoreV2_2/shape_and_slices)]]\n\t [[Node: save_6/RestoreV2_10/_15 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device_incarnation=1, tensor_name=\"edge_160_save_6/RestoreV2_10\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n\nCaused by op u'save_6/RestoreV2_2', defined at:\n  File \"/usr/lib/python2.7/runpy.py\", line 162, in _run_module_as_main\n    \"__main__\", fname, loader, pkg_name)\n  File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code\n    exec code in run_globals\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python2.7/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/usr/local/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py\", line 162, in start\n    super(ZMQIOLoop, self).start()\n  File \"/usr/local/lib/python2.7/dist-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python2.7/dist-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-12-5d8c121d9cf8>\", line 3, in <module>\n    saver = tf.train.Saver(tf.all_variables())\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 1078, in __init__\n    self.build()\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 1107, in build\n    restore_sequentially=self._restore_sequentially)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 705, in build\n    restore_sequentially, reshape)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 442, in _AddRestoreOps\n    tensors = self.restore_op(filename_tensor, saveable, preferred_shard)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/saver.py\", line 281, in restore_op\n    [spec.tensor.dtype])[0])\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_io_ops.py\", line 439, in restore_v2\n    dtypes=dtypes, name=name)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py\", line 749, in apply_op\n    op_def=op_def)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 2386, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 1298, in __init__\n    self._traceback = _extract_stack()\n\nNotFoundError (see above for traceback): Tensor name \"embedding\" not found in checkpoint files wmt/translate.ckpt-176800\n\t [[Node: save_6/RestoreV2_2 = RestoreV2[dtypes=[DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/cpu:0\"](_recv_save_6/Const_0, save_6/RestoreV2_2/tensor_names, save_6/RestoreV2_2/shape_and_slices)]]\n\t [[Node: save_6/RestoreV2_10/_15 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/gpu:0\", send_device=\"/job:localhost/replica:0/task:0/cpu:0\", send_device_incarnation=1, tensor_name=\"edge_160_save_6/RestoreV2_10\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.ops import variable_scope\n",
    "# model = create_model()\n",
    "saver = tf.train.Saver(tf.all_variables())\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"wmt/translate.ckpt-176800\")\n",
    "    train_writer = tf.train.SummaryWriter('/tensorboard', sess.graph)\n",
    "    with tf.variable_scope(\"embedding_rnn_seq2seq\"):\n",
    "        with tf.variable_scope(\"RNN\"):\n",
    "            with tf.variable_scope(\"EmbeddingWrapper\", reuse=True):\n",
    "                embedding = variable_scope.get_variable(\"embedding\", [shape])\n",
    "                print embedding\n",
    "                emb = np.array(embedding)\n",
    "                print emb.shape\n",
    "                f = open(\"wmt/embedding.txt\", 'w')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "V_en = 50000; V_fr = 70000\n",
    "vocab_file = \"wmt/vocab\"+str(V_fr)+\".fr\"\n",
    "data_file = \"wmt/giga-fren.release2.fixed.fr\"\n",
    "target_path = \"wmt/giga-fren.release2.fixed.ids\"+str(V_fr)+\".fr\"\n",
    "\n",
    "dev_data = \"wmt/dev/newstest2013-ref.fr.sgm\"\n",
    "dev_target = \"wmt/dev/newstest2013.ids\"+str(V_fr)+\".fr\"\n",
    "# create_vocabulary(vocab_file, data_file, V_fr)\n",
    "data_to_token_ids(data_file, target_path, vocab_file, tokenizer=None, normalize_digits=True)\n",
    "data_to_token_ids(dev_data, dev_target, vocab_file, tokenizer=None, normalize_digits=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}