train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from kaggle_speech_recog import *\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get feature and label matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "h:\\python35\\lib\\site-packages\\scipy\\io\\wavfile.py:273: WavFileWarning: Chunk (non-data) not understood, skipping it.\n",
      "  WavFileWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SKIPPED ../data/train/audio/_background_noise_/README.md File format b'# Ba'... not understood.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "h:\\python35\\lib\\site-packages\\librosa\\filters.py:261: UserWarning: Empty filters detected in mel frequency basis. Some channels will produce empty responses. Try increasing your sampling rate (and fmax) or reducing n_mels.\n",
      "  warnings.warn('Empty filters detected in mel frequency basis. '\n"
     ]
    }
   ],
   "source": [
    "X_vector_len = 16000\n",
    "train = SpeechList.get_train('../data/train/audio')\n",
    "XY_train_valid = train.get_group_unknown_spectrogram_X_and_Y(X_vector_len=X_vector_len, spec_v='3', take_log=False, \n",
    "                                                             split_noise=True, n_fabricate_noise=1600, \n",
    "                                                             split=0.85)\n",
    "\n",
    "# If using group_unknown, the output is: X_train_known, X_train_unknown, Y_train_known, Y_train_unknown, X_valid, Y_valid\n",
    "# Otherwise: X_train, Y_train, X_valid, Y_valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1701, 128, 71)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "X_train, X_train_unknown, Y_train, Y_train_unknown, X_valid, Y_valid=XY_train_valid\n",
    "print(X_train.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function\n",
    "# def get_shape_after_skipping(initial_shape, strides_list):\n",
    "#     h, w = initial_shape\n",
    "#     for strides in strides_list:\n",
    "#         h = math.ceil(h / strides[0])\n",
    "#         w = math.ceil(w / strides[1])\n",
    "#     return h, w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need this while testing graph so it gets autoloaded\n",
    "from kaggle_speech_recog.graphs.useful_tf_graph import *\n",
    "from kaggle_speech_recog.graphs.spectrogram_multi_lstm_random_input_modify_v2 import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_name = 'SpectrogramMultiLSTMRandomInputModify2'  # Specify graph class name\n",
    "\n",
    "g_cnfg = Config('graph_02_lower_lr_lr_decay')  # Specify graph configuration name\n",
    "g_cnfg.X_img_h = XY_train_valid[0].shape[1]\n",
    "g_cnfg.X_img_w = XY_train_valid[0].shape[2]\n",
    "g_cnfg.Y_vector_len = XY_train_valid[-1].shape[1]\n",
    "\n",
    "g_cnfg.random_modify_brightness_max_delta = 0.25\n",
    "g_cnfg.random_modify_contrast_lower = 0.2\n",
    "g_cnfg.random_modify_contrast_upper = 1.8\n",
    "\n",
    "g_cnfg.lstm_state_sizes = [1024, 1024]\n",
    "g_cnfg.lstm_keep_prob = 0.75\n",
    "\n",
    "g_cnfg.flat_hiddens = [512, 512, 50, 50]\n",
    "g_cnfg.n_hidden1=512\n",
    "g_cnfg.n_hidden2=512\n",
    "g_cnfg.sigmoid_accuracy_threshold=0.1\n",
    "#g_cnfg.conv1_keep_prob = 0.9\n",
    "#g_cnfg.conv1_kernel_size = (10, 10)  # height, width\n",
    "#g_cnfg.conv1_n_filters = 128\n",
    "#g_cnfg.conv1_mp_size = (2, 1)\n",
    "#g_cnfg.conv1_mp_strides = (2, 1)\n",
    "\n",
    "#g_cnfg.conv2ds_keep_prob = 0.9\n",
    "#g_cnfg.conv2ds_kernel_size = (3, 3)\n",
    "#g_cnfg.conv2ds_strides = (1, 1)\n",
    "#g_cnfg.conv2ds_n_filters = 128\n",
    "\n",
    "#strides_list = [g_cnfg.conv1_mp_strides, g_cnfg.conv2ds_strides]\n",
    "#g_cnfg.n_per_time_after_conv = get_shape_after_skipping((g_cnfg.X_img_h, g_cnfg.X_img_w), strides_list)[0] * g_cnfg.conv2ds_n_filters\n",
    "#g_cnfg.n_per_time_after_reducing_conv = 128\n",
    "\n",
    "g_cnfg.lr_initial = 0.01\n",
    "g_cnfg.lr_decay_steps = 50000\n",
    "g_cnfg.lr_decay_rate = 0.1  # 1.0 means no decay\n",
    "\n",
    "t_cnfg = Config('run_01_group_unknown')  # Specify train configuration name\n",
    "t_cnfg.ckp_dir = 'checkpoints'\n",
    "t_cnfg.tb_dir = 'tensorboard'\n",
    "t_cnfg.log_dir = 'logs'\n",
    "t_cnfg.le = train.le\n",
    "t_cnfg.random_pick_unknown = True\n",
    "t_cnfg.max_step = 100000000\n",
    "t_cnfg.batch_size = 150\n",
    "t_cnfg.add_noise = (0.5, [s.spec_data for s in train.get_list_of_label(['_background_noise_'])])\n",
    "t_cnfg.dropout_keep_prob = 0.75\n",
    "t_cnfg.log_every = int(int(XY_train_valid[0].shape[0] / 11 * 12) / t_cnfg.batch_size / 2)\n",
    "t_cnfg.n_ave_ll_valid = 8\n",
    "t_cnfg.start_step_early_stopping = 100000000\n",
    "t_cnfg.early_stopping_patience =0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: graph_02_lower_lr_lr_decay\n",
      "X_img_h: 128\n",
      "X_img_w: 71\n",
      "Y_vector_len: 2\n",
      "flat_hiddens: [512, 512, 50, 50]\n",
      "lr_decay_rate: 0.1\n",
      "lr_decay_steps: 50,000\n",
      "lr_initial: 0.01\n",
      "lstm_keep_prob: 0.75\n",
      "lstm_state_sizes: [1024, 1024]\n",
      "n_hidden1: 512\n",
      "n_hidden2: 512\n",
      "random_modify_brightness_max_delta: 0.25\n",
      "random_modify_contrast_lower: 0.2\n",
      "random_modify_contrast_upper: 1.8\n",
      "sigmoid_accuracy_threshold: 0.1\n",
      "\n",
      "name: run_01_group_unknown\n",
      "add_noise: (0.5, 'list of length 2,002')\n",
      "batch_size: 150\n",
      "ckp_dir: checkpoints\n",
      "dropout_keep_prob: 0.75\n",
      "early_stopping_patience: 0.1\n",
      "le: LabelEncoder()\n",
      "log_dir: logs\n",
      "log_every: 6\n",
      "max_step: 100,000,000\n",
      "n_ave_ll_valid: 8\n",
      "random_pick_unknown: 1\n",
      "start_step_early_stopping: 100,000,000\n",
      "tb_dir: tensorboard\n",
      "\n",
      "graph_name:  SpectrogramMultiLSTMRandomInputModify2\n",
      "graph_name :  SpectrogramMultiLSTMRandomInputModify2\n",
      "self.X:  (?, 128, 71)\n",
      "self.Y:  (?, 2)\n",
      "pickle url  :  logs/SpectrogramMultiLSTMRandomInputModify2_graph_02_lower_lr_lr_decay_run_01_group_unknown.log\n",
      "============================================================\n",
      "SpectrogramMultiLSTMRandomInputModify2_graph_02_lower_lr_lr_decay_run_01_group_unknown\n",
      "============================================================\n",
      "Epoch size is 1,855 | Batch size is 150 | 12 steps per epoch\n",
      "55 leftover gets discarded at the end of every epoch\n",
      "\n",
      "Training starts @ 04/08/2018 12:45:54\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-14.0109383  -10.84023057  -9.36881279 ... -13.60699552 -14.74145535\n",
      "  -11.92708268]\n",
      " ...\n",
      " [-20.12768105 -20.61430414 -17.8820098  ... -17.13350555 -16.24580202\n",
      "  -15.55908641]\n",
      " [-17.6207334  -18.32447847 -18.3075291  ... -15.20955847 -14.61103923\n",
      "  -20.8641557 ]\n",
      " [-19.58909213 -17.74552994 -20.06522188 ... -16.25743371 -13.7309955\n",
      "  -21.22110454]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "pickle url  :  logs/SpectrogramMultiLSTMRandomInputModify2_graph_02_lower_lr_lr_decay_run_01_group_unknown.log\n",
      "Epoch 1 Step 1 Best model saved @ 04/08/2018 12:47:52 [Ave valid] 0.695\n",
      "Epoch 1 Step 1 ends @ 04/08/2018 12:47:52 [Train] 0.695, 10.7% [Valid] 8.2% [Ave valid] 0.695\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -48.38376118  -50.02999166  -54.91136678 ...  -34.65777336\n",
      "   -39.96350124  -31.08176168]\n",
      " ...\n",
      " [ -76.26060049  -77.19924095  -82.23683265 ...  -81.4430443\n",
      "   -80.76395783  -76.71512984]\n",
      " [ -75.74323078  -84.50514219  -82.02717487 ...  -80.59857336\n",
      "   -76.57038919  -79.11866145]\n",
      " [ -77.16930569  -80.70108126  -83.81783186 ...  -82.7610272\n",
      "   -82.33357478  -80.79206315]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -12.23684299  -18.3228093   -22.52970182 ...  -47.19305091\n",
      "   -19.20035758  -21.37940228]\n",
      " ...\n",
      " [ -58.93821891  -65.9058058   -60.88086806 ...  -59.58745666\n",
      "   -64.84220436  -69.63361246]\n",
      " [ -64.08199484  -64.2548739   -64.35824961 ...  -59.22927556\n",
      "   -60.44664203  -67.41938109]\n",
      " [ -55.01580787  -60.03794276  -62.7820968  ...  -63.55723524\n",
      "   -66.47140695  -66.39109671]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-22.79250336 -22.30791768 -19.46235396 ... -20.04125443 -21.22681019\n",
      "  -21.39673656]\n",
      " ...\n",
      " [-36.38873553 -36.9500584  -33.09038492 ... -32.14364623 -36.16537121\n",
      "  -32.92058552]\n",
      " [-37.25991576 -34.78176422 -29.17971349 ... -35.04576068 -36.924267\n",
      "  -39.12777559]\n",
      " [-36.84379614 -33.98651798 -28.54964543 ... -35.13592629 -37.00414087\n",
      "  -38.41226401]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [ -8.46915445  -7.23365048 -17.19767515 ... -11.0265923  -12.24350484\n",
      "  -19.02411942]\n",
      " ...\n",
      " [-29.40808536 -30.38941053 -30.13661184 ... -35.84680018 -33.36844582\n",
      "  -35.67048815]\n",
      " [-31.41755546 -34.69802757 -35.17706713 ... -32.44999606 -34.62553016\n",
      "  -32.34410319]\n",
      " [-31.91124819 -35.9784412  -31.34291204 ... -33.91014786 -33.12302458\n",
      "  -33.28854275]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [ -1.09604646  -2.52108843  -9.69585825 ...  -6.96280704  -5.06890548\n",
      "  -16.1481845 ]\n",
      " ...\n",
      " [-23.39665312 -24.81688942 -26.11576472 ... -21.24813612 -27.82817091\n",
      "  -25.08562602]\n",
      " [-28.23852152 -21.25743229 -22.83541083 ... -26.37685587 -22.36800756\n",
      "  -25.71566651]\n",
      " [-25.13253041 -22.58231303 -23.26490597 ... -28.65132901 -22.86275529\n",
      "  -25.4630273 ]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "pickle url  :  logs/SpectrogramMultiLSTMRandomInputModify2_graph_02_lower_lr_lr_decay_run_01_group_unknown.log\n",
      "Epoch 1 Step 6 Best model saved @ 04/08/2018 12:53:13 [Ave valid] 0.687\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-31.18656434 -11.57117663 -18.66627102 ... -19.61041153 -13.71044256\n",
      "  -21.52985766]\n",
      " ...\n",
      " [-36.03211259 -39.2833975  -38.33588827 ... -40.61142323 -41.3422452\n",
      "  -37.32242604]\n",
      " [-36.10575839 -38.55501152 -40.76668941 ... -37.37264269 -39.08958109\n",
      "  -44.07413161]\n",
      " [-41.34623289 -40.27165135 -38.31881858 ... -42.00649465 -44.29839094\n",
      "  -47.53672113]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -90.10556039  -88.08165613  -84.70598673 ...  -99.01126999\n",
      "   -85.79164038  -83.36070172]\n",
      " ...\n",
      " [ -97.95375921  -87.58592894  -81.39656029 ... -109.76281438\n",
      "  -111.01151962 -111.31511017]\n",
      " [ -97.76799508  -95.11321117  -81.70544278 ... -109.97091573\n",
      "  -110.14748279 -110.53706933]\n",
      " [-103.78521719 -102.32562622  -90.8124744  ... -115.32581192\n",
      "  -111.86752273 -111.78355178]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [0. 1.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-57.09166439 -54.74324312 -58.45077007 ... -53.13989839 -56.06698773\n",
      "  -45.74564084]\n",
      " ...\n",
      " [-72.59725581 -77.59181203 -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [0. 1.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -41.70662721  -34.58339579  -34.22609788 ...  -35.89901488\n",
      "   -32.46278008  -55.61572782]\n",
      " ...\n",
      " [ -67.15461473  -67.61930873  -61.95745787 ...  -62.4168928\n",
      "   -67.62522134  -64.79793002]\n",
      " [ -70.18567484  -65.74485414  -65.83389556 ...  -63.95861464\n",
      "   -68.58517177  -66.98613973]\n",
      " [ -69.86762108  -63.79723339  -67.76829743 ...  -69.53145312\n",
      "   -65.15533283  -70.75617527]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -30.51323884  -41.66873505  -39.28366378 ...  -28.90506011\n",
      "   -37.73771447  -34.54892786]\n",
      " ...\n",
      " [ -51.57247871  -52.20523903  -47.47947049 ...  -52.95238887\n",
      "   -51.94224153  -51.90251579]\n",
      " [ -56.37208482  -48.46343479  -53.59532834 ...  -47.33593401\n",
      "   -51.51658679  -51.96819161]\n",
      " [ -52.05820115  -48.2250777   -46.38479231 ...  -48.61301109\n",
      "   -49.15992692  -57.6239436 ]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-14.24895755  -6.40513465 -17.06170697 ... -12.0744168  -13.47523494\n",
      "  -29.24423778]\n",
      " ...\n",
      " [-18.17848917 -19.61006108 -22.02183623 ... -22.57440831 -23.03530265\n",
      "  -20.61820064]\n",
      " [-21.96448534 -20.50291564 -24.96051694 ... -21.91709458 -23.78297209\n",
      "  -24.34123937]\n",
      " [-20.66086404 -21.95023046 -23.55572991 ... -23.48586079 -18.04966402\n",
      "  -22.2206794 ]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "pickle url  :  logs/SpectrogramMultiLSTMRandomInputModify2_graph_02_lower_lr_lr_decay_run_01_group_unknown.log\n",
      "Epoch 1 Step 12 Best model saved @ 04/08/2018 12:59:25 [Ave valid] 0.679\n",
      "Epoch 1 Step 12 ends @ 04/08/2018 12:59:25 [Train] 0.658, 97.3% [Valid] 91.8% [Ave valid] 0.679\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -28.30805992   -8.08528404  -25.15896889 ...  -28.04178264\n",
      "   -38.44959807  -12.02250493]\n",
      " ...\n",
      " [ -70.75706486  -66.39953975  -62.99685745 ...  -63.64873625\n",
      "   -63.63986081  -72.19176086]\n",
      " [ -66.92942492  -67.36503133  -67.18009145 ...  -68.33732695\n",
      "   -62.89661157  -68.72662545]\n",
      " [ -65.54978443  -63.7062236   -68.55056626 ...  -72.44003555\n",
      "   -69.3629911   -70.11867942]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [ -9.26542252  -9.47530176  -9.5957989  ... -12.06301618 -13.46535939\n",
      "  -21.95397946]\n",
      " ...\n",
      " [-41.62362954 -48.14604885 -43.16860577 ... -44.34860379 -45.5717487\n",
      "  -46.46639053]\n",
      " [-43.1385798  -45.69612034 -45.40441625 ... -43.83268374 -45.48385675\n",
      "  -44.44051409]\n",
      " [-41.50461206 -45.30691723 -44.97069265 ... -45.52738403 -51.10481958\n",
      "  -42.53159868]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -25.23953725  -22.24902131  -24.75842617 ...  -19.00081517\n",
      "   -14.91922382  -30.9687486 ]\n",
      " ...\n",
      " [ -43.74073132  -39.25356965  -46.80948803 ...  -55.52666748\n",
      "   -44.60545058  -47.37149166]\n",
      " [ -39.82954617  -43.0473961   -48.85238536 ...  -47.91081296\n",
      "   -46.42298726  -47.67952899]\n",
      " [ -43.86900838  -48.13381248  -44.47899247 ...  -46.43276249\n",
      "   -43.73664084  -45.70579032]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -33.57412784  -23.12846593  -52.35660516 ...  -51.62957604\n",
      "   -45.14948876  -45.34901332]\n",
      " ...\n",
      " [ -47.91672509  -24.22391857  -30.63531401 ...  -56.82912601\n",
      "   -58.39356075  -54.98888862]\n",
      " [ -42.51839047  -22.28016711  -24.26033656 ...  -59.20581543\n",
      "   -56.27891406  -55.52128854]\n",
      " [ -40.62055806  -23.95337542  -19.33920443 ...  -55.3940599\n",
      "   -60.52167484  -56.16115754]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-13.88536328 -17.12555882 -21.01580448 ... -22.31507537 -26.01905592\n",
      "  -17.59060179]\n",
      " ...\n",
      " [-20.64765422 -18.18828157 -24.94900358 ... -19.02232017 -19.9188688\n",
      "  -20.65613117]\n",
      " [-26.70579905 -20.22466131 -21.81673465 ... -21.99618966 -23.30676771\n",
      "  -20.64103193]\n",
      " [-25.6945546  -18.31440915 -21.41860525 ... -26.33189492 -25.48936968\n",
      "  -21.69302318]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -34.14316114  -20.66807359  -51.54302359 ...  -30.384018\n",
      "   -19.78218347  -17.2907046 ]\n",
      " ...\n",
      " [ -38.45267519  -35.81909032  -28.14473868 ...  -36.44759011\n",
      "   -39.43909931  -30.59550755]\n",
      " [ -36.07024692  -35.51008165  -33.50315436 ...  -35.53683618\n",
      "   -39.71550418  -32.45878267]\n",
      " [ -35.02174306  -42.03527985  -37.27737649 ...  -39.9401815\n",
      "   -34.34505545  -33.31758355]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "pickle url  :  logs/SpectrogramMultiLSTMRandomInputModify2_graph_02_lower_lr_lr_decay_run_01_group_unknown.log\n",
      "Epoch 2 Step 18 Best model saved @ 04/08/2018 13:05:47 [Ave valid] 0.671\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-16.87754416 -10.33565537  -5.86906937 ... -14.17104901  -8.05544704\n",
      "  -23.81102008]\n",
      " ...\n",
      " [-15.70624433 -14.78754491 -13.82753167 ... -14.58609486 -14.90480219\n",
      "  -12.32689346]\n",
      " [-11.69132952  -8.99368013 -14.61647773 ... -12.13501407 -10.90924259\n",
      "  -11.85469099]\n",
      " [-15.21718916 -10.19109029 -14.18780829 ... -12.00698118 -12.87428827\n",
      "   -9.64892926]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -38.87667253  -33.94640794  -37.14608559 ...  -16.99797987\n",
      "   -18.29903017  -29.72110992]\n",
      " ...\n",
      " [ -51.96460427  -58.3126624   -62.86542166 ...  -61.07517498\n",
      "   -63.88360431  -55.51694234]\n",
      " [ -54.32316055  -60.3346643   -65.31302628 ...  -60.48066301\n",
      "   -63.64802997  -58.08761093]\n",
      " [ -59.57713602  -61.06313602  -64.84483366 ...  -62.30964935\n",
      "   -66.48851899  -60.1264736 ]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [ -9.90607629  -6.39000522 -11.74558068 ... -14.19839422 -15.89720681\n",
      "  -11.18237978]\n",
      " ...\n",
      " [-31.71174823 -31.3308434  -34.00268086 ... -30.26823029 -28.96481735\n",
      "  -34.26693586]\n",
      " [-34.37466514 -30.94895512 -33.65402291 ... -29.29553301 -33.78274253\n",
      "  -32.88195929]\n",
      " [-34.81161762 -33.53981989 -36.78963503 ... -30.1452901  -32.37644425\n",
      "  -31.72248305]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -10.21239132  -36.31437376  -32.54714176 ...  -20.50802502\n",
      "   -22.21309851  -23.11943339]\n",
      " ...\n",
      " [ -38.85965323  -35.33014651  -36.01155169 ...  -39.4463583\n",
      "   -41.39054516  -39.81135293]\n",
      " [ -42.31686318  -36.77966152  -32.02655622 ...  -35.02027163\n",
      "   -36.92214351  -38.13458835]\n",
      " [ -37.564728    -38.6568044   -36.91369839 ...  -39.54301791\n",
      "   -37.24721847  -42.41052296]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -31.58964434  -34.49101518  -26.19040649 ...  -40.85464613\n",
      "   -35.20180368  -30.28467458]\n",
      " ...\n",
      " [ -51.08327706  -46.65584031  -54.80481376 ...  -47.76249525\n",
      "   -46.39751484  -46.31290563]\n",
      " [ -49.57988219  -49.72734119  -53.88493934 ...  -59.45391948\n",
      "   -48.25658608  -51.96002493]\n",
      " [ -47.59094555  -51.56353054  -48.77340699 ...  -54.34386373\n",
      "   -47.60092502  -51.6972032 ]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [-80.         -80.         -80.         ... -80.         -80.\n",
      "  -80.        ]\n",
      " [ -4.07081603 -12.92215418 -19.50738652 ... -15.78709033 -16.21270198\n",
      "  -37.11869049]\n",
      " ...\n",
      " [-29.00891824 -32.62850747 -35.4082781  ... -33.56975913 -33.78065355\n",
      "  -36.59114323]\n",
      " [-33.59271676 -34.41665313 -35.5837693  ... -33.72262112 -35.57811893\n",
      "  -37.69061945]\n",
      " [-31.87713472 -37.10094796 -37.88122352 ... -38.51019177 -36.08845735\n",
      "  -41.50571163]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pickle url  :  logs/SpectrogramMultiLSTMRandomInputModify2_graph_02_lower_lr_lr_decay_run_01_group_unknown.log\n",
      "Epoch 2 Step 24 Best model saved @ 04/08/2018 13:12:01 [Ave valid] 0.663\n",
      "Epoch 2 Step 24 ends @ 04/08/2018 13:12:01 [Train] 0.631, 93.3% [Valid] 91.8% [Ave valid] 0.663\n",
      "X_batch:  (150, 128, 71)\n",
      "X_batch:  [[-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [-160.         -160.         -160.         ... -160.\n",
      "  -160.         -160.        ]\n",
      " [ -33.97076819  -42.29044679  -39.38466923 ...  -48.63222005\n",
      "   -41.21467985  -38.41412872]\n",
      " ...\n",
      " [ -42.31733669  -53.32206964  -47.64145048 ...  -48.41659799\n",
      "   -46.69459285  -50.16444542]\n",
      " [ -48.55732226  -50.33942784  -43.56796743 ...  -45.02822709\n",
      "   -43.7114746   -46.61289167]\n",
      " [ -47.03897102  -49.35890898  -45.33406305 ...  -43.75410491\n",
      "   -49.86018068  -43.22223904]]\n",
      "Y_batch:  (150, 2)\n",
      "Y_batch:  [1. 0.]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-966db41cd142>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mgraph\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mGraphClass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mg_cnfg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mt_cnfg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mXY_train_valid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mH:\\PycharmProjects\\speech-recognition-kumikokashii\\working\\kaggle_speech_recog\\graphs\\useful_tf_graph.py\u001b[0m in \u001b[0;36mtrain_model\u001b[1;34m(self, cnfg, XY_train_valid, annotate)\u001b[0m\n\u001b[0;32m     81\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Y_batch: '\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mY_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m                 _, summary = self.sess.run([self.optimizer, self.summarizer], feed_dict={self.X: X_batch, self.Y: Y_batch, \n\u001b[1;32m---> 83\u001b[1;33m                                                       self.keep_prob: cnfg.dropout_keep_prob, self.is_training: True})\n\u001b[0m\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moffset\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mcnfg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_every\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# Keep track of training progress\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mh:\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    887\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 889\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    890\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mh:\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1118\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1120\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1121\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mh:\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1315\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1317\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1318\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1319\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mh:\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1321\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1322\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1323\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1324\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1325\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mh:\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1300\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[0;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1302\u001b[1;33m                                    status, run_metadata)\n\u001b[0m\u001b[0;32m   1303\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1304\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "print(g_cnfg)\n",
    "print(t_cnfg)\n",
    "print('graph_name: ',graph_name)\n",
    "GraphClass = globals()[graph_name]\n",
    "print('graph_name : ',graph_name)\n",
    "graph = GraphClass(g_cnfg)\n",
    "\n",
    "graph.train_model(t_cnfg, XY_train_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}