GNN_stacking_0815.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "from torch_geometric.data import Data,DataLoader\n",
    "from functions_refactor import *\n",
    "from pytorch_util import *\n",
    "#from torch.optim import Adam\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fixed parameters\n",
    "block = MEGNet_block\n",
    "head_mol,head_atom,head_edge = head_mol,head_atom,head_edge\n",
    "clip = 0.5\n",
    "batch_size = 64\n",
    "threshold = 1e3\n",
    "reuse = False\n",
    "lr = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# changing parameters\n",
    "head = SimplyInteraction\n",
    "data = '../Data/{}_data_stacking_0815.pickle'\n",
    "dim = 22\n",
    "logLoss = False\n",
    "layer = 4\n",
    "factor = 3\n",
    "epochs = 19\n",
    "aggr = 'max'\n",
    "sample_rate = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(data.format('train'), 'rb') as handle:\n",
    "    train_data = pickle.load(handle)\n",
    "with open(data.format('val'), 'rb') as handle:\n",
    "    val_data = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = train_data + val_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_feature(dict_,sample_index):\n",
    "    dict_copy = copy.deepcopy(dict_)\n",
    "    dict_copy['edge_attr4'] = dict_copy['edge_attr4'][:,sample_index]\n",
    "    dict_copy['x'] = dict_copy['x'][:,sample_index]\n",
    "    return dict_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('../Data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GNN_multiHead_interleave_stacking(torch.nn.Module):\n",
    "    def __init__(self,reuse,block,head,dim,layer,factor,\\\n",
    "                 edge_in3=8,aggr='mean'):\n",
    "        # block,head are nn.Module\n",
    "        # node_in,edge_in are dim for bonding and edge_in4,edge_in3 for coupling\n",
    "        super(GNN_multiHead_interleave_stacking, self).__init__()\n",
    "       \n",
    "        if reuse:\n",
    "            self.conv = block(dim=dim,aggr=aggr)\n",
    "        else:\n",
    "            self.conv = nn.ModuleList([block(dim=dim,aggr=aggr) for _ in range(layer)])        \n",
    "        self.head = head(dim)\n",
    "        \n",
    "        \n",
    "    def forward(self, data,IsTrain=False,typeTrain=False,logLoss=True,weight=None):\n",
    "        out = data.x\n",
    "        # edge_*3 only does not repeat for undirected graph. Hence need to add (j,i) to (i,j) in edges\n",
    "        edge_index3 = torch.cat([data.edge_index3,data.edge_index3[[1,0]]],1)\n",
    "        n = data.edge_attr3.shape[0]\n",
    "        edge_attr3 = torch.cat([data.edge_attr4,data.edge_attr4],0)\n",
    "          \n",
    "        for conv in self.conv:\n",
    "            out,edge_attr3 = conv(out,edge_index3,edge_attr3)\n",
    "\n",
    "        \n",
    "        edge_attr3 = edge_attr3[:n]\n",
    "\n",
    "        if typeTrain:\n",
    "            if IsTrain:\n",
    "                y = data.y[data.type_attr]\n",
    "            edge_attr3 = edge_attr3[data.type_attr]\n",
    "            edge_index3 = data.edge_index3[:,data.type_attr]\n",
    "            edge_attr3_old = data.edge_attr3[data.type_attr]\n",
    "        else:\n",
    "            if IsTrain:\n",
    "                y = data.y\n",
    "            edge_index3 = data.edge_index3\n",
    "            edge_attr3_old = data.edge_attr3\n",
    "            \n",
    "        yhat = self.head(out,edge_index3,edge_attr3,edge_attr3_old)\n",
    "        \n",
    "        if IsTrain:\n",
    "            k = torch.sum(edge_attr3_old,0)\n",
    "            nonzeroIndex = torch.nonzero(k).squeeze(1)\n",
    "            abs_ = torch.abs(y-yhat).unsqueeze(1)\n",
    "            loss_perType = torch.zeros(8,device='cuda:0')\n",
    "            if logLoss:\n",
    "                loss_perType[nonzeroIndex] = torch.log(torch.sum(abs_ * edge_attr3_old[:,nonzeroIndex],0)/k[nonzeroIndex])\n",
    "                loss = torch.sum(loss_perType)/nonzeroIndex.shape[0]\n",
    "                return loss,loss_perType         \n",
    "            else:\n",
    "                loss_perType[nonzeroIndex] = torch.sum(abs_ * edge_attr3_old[:,nonzeroIndex],0)/k[nonzeroIndex]\n",
    "                loss = torch.sum(loss_perType)/nonzeroIndex.shape[0]\n",
    "                loss_perType[nonzeroIndex] = torch.log(loss_perType[nonzeroIndex])\n",
    "                return loss,loss_perType\n",
    "        else:\n",
    "            return yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----start bag: 0-----\n",
      "epoch:0, train_loss: +5.435, val_loss: -2.408, \n",
      "train_vector: +1.03|+0.69|-1.20|-1.36|-1.77|-0.87|-0.92|-2.25, \n",
      "val_vector  : -1.54|-1.76|-2.38|-2.65|-2.90|-2.17|-2.83|-3.03\n",
      "\n",
      "epoch:1, train_loss: +0.105, val_loss: -2.465, \n",
      "train_vector: -1.53|-1.67|-2.44|-2.68|-2.97|-2.24|-2.95|-3.10, \n",
      "val_vector  : -1.54|-1.94|-2.48|-2.34|-2.98|-2.30|-3.04|-3.09\n",
      "\n",
      "epoch:2, train_loss: +0.096, val_loss: -2.572, \n",
      "train_vector: -1.52|-1.83|-2.51|-2.91|-3.01|-2.33|-3.07|-3.17, \n",
      "val_vector  : -1.46|-1.88|-2.51|-3.07|-3.00|-2.35|-3.12|-3.18\n",
      "\n",
      "epoch:3, train_loss: +0.093, val_loss: -2.621, \n",
      "train_vector: -1.54|-1.88|-2.53|-3.04|-3.03|-2.36|-3.11|-3.19, \n",
      "val_vector  : -1.55|-1.92|-2.55|-3.20|-3.03|-2.37|-3.15|-3.20\n",
      "\n",
      "epoch:4, train_loss: +0.091, val_loss: -2.634, \n",
      "train_vector: -1.55|-1.90|-2.55|-3.10|-3.03|-2.37|-3.13|-3.21, \n",
      "val_vector  : -1.56|-1.95|-2.54|-3.28|-3.03|-2.37|-3.14|-3.20\n",
      "\n",
      "epoch:5, train_loss: +0.089, val_loss: -2.603, \n",
      "train_vector: -1.57|-1.94|-2.56|-3.12|-3.05|-2.39|-3.15|-3.22, \n",
      "val_vector  : -1.52|-1.97|-2.53|-3.19|-3.02|-2.36|-3.07|-3.16\n",
      "\n",
      "epoch:6, train_loss: +0.089, val_loss: -2.637, \n",
      "train_vector: -1.57|-1.93|-2.57|-3.15|-3.05|-2.39|-3.15|-3.22, \n",
      "val_vector  : -1.60|-2.05|-2.57|-3.11|-3.01|-2.39|-3.16|-3.21\n",
      "\n",
      "epoch:7, train_loss: +0.089, val_loss: -2.583, \n",
      "train_vector: -1.58|-1.91|-2.57|-3.18|-3.05|-2.40|-3.16|-3.23, \n",
      "val_vector  : -1.59|-1.95|-2.56|-2.81|-3.05|-2.37|-3.11|-3.23\n",
      "\n",
      "epoch:8, train_loss: +0.088, val_loss: -2.548, \n",
      "train_vector: -1.57|-1.92|-2.57|-3.22|-3.05|-2.40|-3.16|-3.23, \n",
      "val_vector  : -1.51|-2.05|-2.57|-2.50|-3.04|-2.38|-3.16|-3.17\n",
      "\n",
      "epoch:9, train_loss: +0.088, val_loss: -2.610, \n",
      "train_vector: -1.58|-1.91|-2.58|-3.24|-3.06|-2.40|-3.16|-3.23, \n",
      "val_vector  : -1.46|-1.93|-2.57|-3.13|-3.05|-2.37|-3.15|-3.21\n",
      "\n",
      "epoch:10, train_loss: +0.089, val_loss: -2.535, \n",
      "train_vector: -1.58|-1.89|-2.58|-3.25|-3.06|-2.40|-3.17|-3.24, \n",
      "val_vector  : -1.57|-2.02|-2.39|-3.32|-2.98|-2.23|-2.73|-3.03\n",
      "\n",
      "epoch:11, train_loss: +0.088, val_loss: -2.503, \n",
      "train_vector: -1.57|-1.89|-2.58|-3.26|-3.06|-2.40|-3.17|-3.23, \n",
      "val_vector  : -1.23|-2.06|-2.47|-2.89|-3.04|-2.31|-2.89|-3.13\n",
      "\n",
      "epoch:12, train_loss: +0.087, val_loss: -2.567, \n",
      "train_vector: -1.58|-1.94|-2.58|-3.28|-3.06|-2.40|-3.18|-3.24, \n",
      "val_vector  : -1.56|-2.07|-2.55|-2.70|-3.02|-2.39|-3.11|-3.14\n",
      "\n",
      "epoch:13, train_loss: +0.085, val_loss: -2.644, \n",
      "train_vector: -1.59|-1.99|-2.59|-3.32|-3.07|-2.41|-3.19|-3.24, \n",
      "val_vector  : -1.61|-1.97|-2.59|-3.15|-3.03|-2.40|-3.15|-3.25\n",
      "\n",
      "epoch:14, train_loss: +0.085, val_loss: -2.626, \n",
      "train_vector: -1.59|-1.99|-2.59|-3.33|-3.07|-2.41|-3.19|-3.24, \n",
      "val_vector  : -1.46|-2.04|-2.59|-3.01|-3.07|-2.40|-3.18|-3.25\n",
      "\n",
      "epoch:15, train_loss: +0.085, val_loss: -2.646, \n",
      "train_vector: -1.59|-1.99|-2.59|-3.34|-3.07|-2.41|-3.19|-3.25, \n",
      "val_vector  : -1.37|-1.92|-2.59|-3.39|-3.07|-2.40|-3.18|-3.25\n",
      "\n",
      "epoch:16, train_loss: +0.085, val_loss: -2.661, \n",
      "train_vector: -1.59|-2.00|-2.59|-3.35|-3.08|-2.41|-3.19|-3.25, \n",
      "val_vector  : -1.44|-2.05|-2.59|-3.35|-3.04|-2.41|-3.19|-3.23\n",
      "\n",
      "epoch:17, train_loss: +0.085, val_loss: -2.574, \n",
      "train_vector: -1.60|-2.00|-2.59|-3.36|-3.08|-2.41|-3.19|-3.25, \n",
      "val_vector  : -1.59|-1.86|-2.53|-3.14|-3.04|-2.35|-2.94|-3.14\n",
      "\n",
      "epoch:18, train_loss: +0.085, val_loss: -2.666, \n",
      "train_vector: -1.59|-2.01|-2.59|-3.35|-3.07|-2.41|-3.20|-3.24, \n",
      "val_vector  : -1.60|-2.06|-2.59|-3.18|-3.07|-2.41|-3.17|-3.24\n",
      "\n",
      "Training completed in 661.5756316184998s\n",
      "-----start bag: 1-----\n",
      "epoch:0, train_loss: +8.480, val_loss: -2.162, \n",
      "train_vector: +2.14|+1.91|-1.02|-0.65|-1.93|-0.49|-0.90|-1.39, \n",
      "val_vector  : -1.26|-0.88|-2.21|-2.83|-2.78|-1.95|-2.62|-2.77\n",
      "\n",
      "epoch:1, train_loss: +0.127, val_loss: -2.506, \n",
      "train_vector: -1.31|-1.36|-2.30|-3.19|-2.86|-2.08|-2.82|-2.96, \n",
      "val_vector  : -1.42|-1.90|-2.41|-3.28|-2.94|-2.17|-2.91|-3.02\n",
      "\n",
      "epoch:2, train_loss: +0.104, val_loss: -2.544, \n",
      "train_vector: -1.39|-1.77|-2.43|-3.28|-2.94|-2.19|-2.97|-3.08, \n",
      "val_vector  : -1.48|-2.02|-2.46|-3.12|-2.96|-2.23|-3.00|-3.09\n",
      "\n",
      "epoch:3, train_loss: +0.098, val_loss: -2.607, \n",
      "train_vector: -1.41|-1.86|-2.48|-3.33|-2.99|-2.25|-3.07|-3.14, \n",
      "val_vector  : -1.51|-2.00|-2.50|-3.36|-2.96|-2.28|-3.10|-3.15\n",
      "\n",
      "epoch:4, train_loss: +0.095, val_loss: -2.570, \n",
      "train_vector: -1.42|-1.92|-2.51|-3.34|-3.01|-2.30|-3.11|-3.18, \n",
      "val_vector  : -1.52|-1.94|-2.51|-3.09|-2.99|-2.31|-3.06|-3.13\n",
      "\n",
      "epoch:5, train_loss: +0.094, val_loss: -2.622, \n",
      "train_vector: -1.41|-1.93|-2.53|-3.35|-3.02|-2.33|-3.13|-3.20, \n",
      "val_vector  : -1.52|-1.98|-2.54|-3.37|-2.97|-2.34|-3.09|-3.16\n",
      "\n",
      "epoch:6, train_loss: +0.092, val_loss: -2.634, \n",
      "train_vector: -1.44|-1.96|-2.54|-3.37|-3.04|-2.35|-3.14|-3.21, \n",
      "val_vector  : -1.52|-2.04|-2.55|-3.32|-3.03|-2.36|-3.14|-3.10\n",
      "\n",
      "epoch:7, train_loss: +0.091, val_loss: -2.640, \n",
      "train_vector: -1.43|-1.99|-2.55|-3.37|-3.04|-2.36|-3.16|-3.22, \n",
      "val_vector  : -1.53|-2.02|-2.55|-3.38|-3.01|-2.37|-3.11|-3.15\n",
      "\n",
      "epoch:8, train_loss: +0.092, val_loss: -2.573, \n",
      "train_vector: -1.41|-1.97|-2.56|-3.37|-3.05|-2.37|-3.16|-3.23, \n",
      "val_vector  : -1.54|-1.99|-2.53|-3.09|-2.97|-2.35|-3.07|-3.04\n",
      "\n",
      "epoch:9, train_loss: +0.091, val_loss: -2.575, \n",
      "train_vector: -1.44|-1.98|-2.56|-3.38|-3.05|-2.37|-3.17|-3.23, \n",
      "val_vector  : -1.11|-2.00|-2.56|-3.39|-3.02|-2.34|-3.11|-3.07\n",
      "\n",
      "epoch:10, train_loss: +0.090, val_loss: -2.555, \n",
      "train_vector: -1.46|-1.99|-2.56|-3.38|-3.06|-2.38|-3.17|-3.23, \n",
      "val_vector  : -1.55|-2.00|-2.51|-3.08|-2.92|-2.39|-3.02|-2.97\n",
      "\n",
      "epoch:11, train_loss: +0.089, val_loss: -2.603, \n",
      "train_vector: -1.48|-2.00|-2.56|-3.39|-3.06|-2.38|-3.17|-3.24, \n",
      "val_vector  : -1.28|-2.06|-2.55|-3.22|-3.03|-2.36|-3.13|-3.20\n",
      "\n",
      "epoch:12, train_loss: +0.088, val_loss: -2.601, \n",
      "train_vector: -1.50|-2.01|-2.57|-3.39|-3.06|-2.38|-3.18|-3.24, \n",
      "val_vector  : -1.11|-1.97|-2.57|-3.32|-3.05|-2.39|-3.17|-3.23\n",
      "\n",
      "epoch:13, train_loss: +0.088, val_loss: -2.609, \n",
      "train_vector: -1.50|-2.03|-2.57|-3.39|-3.07|-2.38|-3.18|-3.24, \n",
      "val_vector  : -1.30|-2.03|-2.56|-3.24|-3.02|-2.34|-3.16|-3.22\n",
      "\n",
      "epoch:14, train_loss: +0.086, val_loss: -2.679, \n",
      "train_vector: -1.53|-2.05|-2.57|-3.41|-3.08|-2.39|-3.19|-3.25, \n",
      "val_vector  : -1.56|-2.04|-2.58|-3.40|-3.07|-2.39|-3.18|-3.22\n",
      "\n",
      "epoch:15, train_loss: +0.085, val_loss: -2.686, \n",
      "train_vector: -1.54|-2.06|-2.57|-3.41|-3.08|-2.39|-3.19|-3.24, \n",
      "val_vector  : -1.56|-2.08|-2.57|-3.40|-3.07|-2.39|-3.18|-3.23\n",
      "\n",
      "epoch:16, train_loss: +0.084, val_loss: -2.668, \n",
      "train_vector: -1.57|-2.07|-2.58|-3.41|-3.08|-2.39|-3.19|-3.25, \n",
      "val_vector  : -1.45|-2.08|-2.58|-3.41|-3.04|-2.39|-3.17|-3.23\n",
      "\n",
      "epoch:17, train_loss: +0.085, val_loss: -2.620, \n",
      "train_vector: -1.56|-2.06|-2.58|-3.41|-3.09|-2.39|-3.19|-3.25, \n",
      "val_vector  : -1.56|-2.09|-2.55|-3.17|-2.99|-2.36|-3.09|-3.16\n",
      "\n",
      "epoch:18, train_loss: +0.085, val_loss: -2.688, \n",
      "train_vector: -1.55|-2.07|-2.58|-3.41|-3.08|-2.39|-3.20|-3.25, \n",
      "val_vector  : -1.60|-2.05|-2.58|-3.40|-3.07|-2.40|-3.18|-3.24\n",
      "\n",
      "Training completed in 670.5826292037964s\n",
      "-----start bag: 2-----\n",
      "epoch:0, train_loss: +6.636, val_loss: -2.407, \n",
      "train_vector: +1.62|+0.48|-1.29|-0.95|-1.60|-0.94|-0.36|-1.69, \n",
      "val_vector  : -1.47|-1.98|-2.30|-2.55|-2.88|-2.17|-2.86|-3.04\n",
      "\n",
      "epoch:1, train_loss: +0.104, val_loss: -2.512, \n",
      "train_vector: -1.42|-1.76|-2.38|-3.02|-2.95|-2.26|-2.93|-3.11, \n",
      "val_vector  : -1.51|-1.45|-2.46|-3.22|-2.96|-2.31|-3.02|-3.16\n",
      "\n",
      "epoch:2, train_loss: +0.098, val_loss: -2.599, \n",
      "train_vector: -1.43|-1.83|-2.47|-3.16|-3.00|-2.32|-3.03|-3.17, \n",
      "val_vector  : -1.45|-2.04|-2.51|-3.23|-3.00|-2.35|-3.02|-3.20\n",
      "\n",
      "epoch:3, train_loss: +0.096, val_loss: -2.588, \n",
      "train_vector: -1.45|-1.85|-2.51|-3.22|-3.01|-2.35|-3.06|-3.19, \n",
      "val_vector  : -1.57|-2.07|-2.49|-3.15|-2.93|-2.36|-3.05|-3.07\n",
      "\n",
      "epoch:4, train_loss: +0.094, val_loss: -2.617, \n",
      "train_vector: -1.47|-1.86|-2.53|-3.26|-3.02|-2.37|-3.09|-3.20, \n",
      "val_vector  : -1.51|-1.93|-2.54|-3.25|-3.02|-2.38|-3.10|-3.20\n",
      "\n",
      "epoch:5, train_loss: +0.093, val_loss: -2.576, \n",
      "train_vector: -1.49|-1.86|-2.55|-3.26|-3.03|-2.38|-3.11|-3.21, \n",
      "val_vector  : -1.24|-1.76|-2.57|-3.35|-2.95|-2.39|-3.14|-3.21\n",
      "\n",
      "epoch:6, train_loss: +0.092, val_loss: -2.576, \n",
      "train_vector: -1.48|-1.89|-2.55|-3.28|-3.04|-2.39|-3.13|-3.22, \n",
      "val_vector  : -1.58|-1.82|-2.50|-3.21|-2.97|-2.36|-3.06|-3.10\n",
      "\n",
      "epoch:7, train_loss: +0.091, val_loss: -2.611, \n",
      "train_vector: -1.49|-1.90|-2.56|-3.30|-3.04|-2.39|-3.14|-3.23, \n",
      "val_vector  : -1.26|-1.96|-2.57|-3.28|-3.03|-2.40|-3.14|-3.25\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:8, train_loss: +0.091, val_loss: -2.585, \n",
      "train_vector: -1.49|-1.90|-2.57|-3.31|-3.05|-2.40|-3.15|-3.23, \n",
      "val_vector  : -1.19|-1.94|-2.57|-3.36|-3.01|-2.37|-3.10|-3.14\n",
      "\n",
      "epoch:9, train_loss: +0.090, val_loss: -2.649, \n",
      "train_vector: -1.50|-1.92|-2.57|-3.32|-3.05|-2.40|-3.15|-3.23, \n",
      "val_vector  : -1.37|-2.08|-2.58|-3.36|-3.01|-2.41|-3.13|-3.24\n",
      "\n",
      "epoch:10, train_loss: +0.090, val_loss: -2.589, \n",
      "train_vector: -1.48|-1.94|-2.57|-3.33|-3.05|-2.40|-3.16|-3.24, \n",
      "val_vector  : -1.47|-1.87|-2.57|-3.22|-2.96|-2.41|-3.08|-3.14\n",
      "\n",
      "epoch:11, train_loss: +0.090, val_loss: -2.656, \n",
      "train_vector: -1.49|-1.90|-2.58|-3.33|-3.05|-2.40|-3.16|-3.23, \n",
      "val_vector  : -1.60|-2.07|-2.58|-3.17|-3.02|-2.40|-3.18|-3.24\n",
      "\n",
      "epoch:12, train_loss: +0.090, val_loss: -2.663, \n",
      "train_vector: -1.50|-1.92|-2.58|-3.33|-3.06|-2.41|-3.17|-3.24, \n",
      "val_vector  : -1.54|-2.07|-2.58|-3.33|-3.05|-2.41|-3.09|-3.24\n",
      "\n",
      "epoch:13, train_loss: +0.089, val_loss: -2.662, \n",
      "train_vector: -1.52|-1.93|-2.58|-3.33|-3.06|-2.41|-3.17|-3.24, \n",
      "val_vector  : -1.52|-2.08|-2.58|-3.23|-3.04|-2.41|-3.18|-3.26\n",
      "\n",
      "epoch:14, train_loss: +0.089, val_loss: -2.592, \n",
      "train_vector: -1.53|-1.92|-2.58|-3.34|-3.06|-2.41|-3.17|-3.24, \n",
      "val_vector  : -1.56|-2.07|-2.54|-2.94|-2.98|-2.39|-3.07|-3.19\n",
      "\n",
      "epoch:15, train_loss: +0.088, val_loss: -2.674, \n",
      "train_vector: -1.53|-1.95|-2.58|-3.35|-3.06|-2.41|-3.17|-3.24, \n",
      "val_vector  : -1.61|-2.09|-2.59|-3.25|-3.00|-2.42|-3.16|-3.27\n",
      "\n",
      "epoch:16, train_loss: +0.088, val_loss: -2.685, \n",
      "train_vector: -1.53|-1.96|-2.58|-3.35|-3.06|-2.41|-3.18|-3.24, \n",
      "val_vector  : -1.60|-2.01|-2.59|-3.36|-3.04|-2.42|-3.20|-3.26\n",
      "\n",
      "epoch:17, train_loss: +0.089, val_loss: -2.652, \n",
      "train_vector: -1.51|-1.91|-2.58|-3.35|-3.05|-2.41|-3.18|-3.24, \n",
      "val_vector  : -1.59|-2.04|-2.58|-3.21|-3.04|-2.40|-3.14|-3.21\n",
      "\n",
      "epoch:18, train_loss: +0.088, val_loss: -2.670, \n",
      "train_vector: -1.53|-1.93|-2.58|-3.34|-3.07|-2.41|-3.18|-3.25, \n",
      "val_vector  : -1.55|-2.06|-2.59|-3.26|-3.05|-2.41|-3.16|-3.27\n",
      "\n",
      "Training completed in 669.6911108493805s\n",
      "-----start bag: 3-----\n",
      "epoch:0, train_loss: +9.214, val_loss: -2.183, \n",
      "train_vector: +1.70|+2.64|-1.02|-1.12|-1.00|-1.16|+0.13|-1.92, \n",
      "val_vector  : -1.01|-1.43|-2.23|-2.76|-2.72|-1.95|-2.42|-2.94\n",
      "\n",
      "epoch:1, train_loss: +0.151, val_loss: -2.490, \n",
      "train_vector: -1.28|-0.93|-2.39|-2.84|-2.84|-2.10|-2.77|-3.04, \n",
      "val_vector  : -1.38|-1.88|-2.47|-3.04|-2.93|-2.19|-2.94|-3.09\n",
      "\n",
      "epoch:2, train_loss: +0.118, val_loss: -2.477, \n",
      "train_vector: -1.40|-1.34|-2.49|-2.93|-2.93|-2.24|-2.97|-3.13, \n",
      "val_vector  : -1.39|-1.44|-2.52|-3.13|-2.95|-2.27|-2.99|-3.13\n",
      "\n",
      "epoch:3, train_loss: +0.108, val_loss: -2.525, \n",
      "train_vector: -1.43|-1.51|-2.53|-3.05|-2.96|-2.30|-3.03|-3.17, \n",
      "val_vector  : -1.46|-1.93|-2.51|-3.06|-2.92|-2.28|-2.99|-3.05\n",
      "\n",
      "epoch:4, train_loss: +0.104, val_loss: -2.542, \n",
      "train_vector: -1.44|-1.58|-2.54|-3.13|-2.98|-2.33|-3.05|-3.19, \n",
      "val_vector  : -1.50|-1.50|-2.52|-3.26|-2.96|-2.32|-3.09|-3.18\n",
      "\n",
      "epoch:5, train_loss: +0.102, val_loss: -2.546, \n",
      "train_vector: -1.45|-1.60|-2.55|-3.18|-2.99|-2.35|-3.07|-3.21, \n",
      "val_vector  : -1.13|-1.95|-2.55|-3.12|-3.01|-2.34|-3.11|-3.16\n",
      "\n",
      "epoch:6, train_loss: +0.099, val_loss: -2.593, \n",
      "train_vector: -1.47|-1.65|-2.56|-3.21|-3.00|-2.36|-3.08|-3.22, \n",
      "val_vector  : -1.47|-1.72|-2.55|-3.31|-3.02|-2.37|-3.11|-3.18\n",
      "\n",
      "epoch:7, train_loss: +0.098, val_loss: -2.558, \n",
      "train_vector: -1.48|-1.68|-2.56|-3.24|-3.01|-2.37|-3.10|-3.22, \n",
      "val_vector  : -1.01|-1.94|-2.56|-3.27|-3.03|-2.36|-3.11|-3.18\n",
      "\n",
      "epoch:8, train_loss: +0.096, val_loss: -2.639, \n",
      "train_vector: -1.49|-1.71|-2.56|-3.27|-3.02|-2.38|-3.11|-3.22, \n",
      "val_vector  : -1.54|-1.98|-2.56|-3.30|-3.02|-2.38|-3.14|-3.20\n",
      "\n",
      "epoch:9, train_loss: +0.094, val_loss: -2.563, \n",
      "train_vector: -1.49|-1.77|-2.57|-3.28|-3.03|-2.39|-3.13|-3.23, \n",
      "val_vector  : -1.43|-2.00|-2.53|-3.08|-2.90|-2.30|-3.15|-3.12\n",
      "\n",
      "epoch:10, train_loss: +0.095, val_loss: -2.628, \n",
      "train_vector: -1.49|-1.73|-2.57|-3.30|-3.03|-2.39|-3.13|-3.23, \n",
      "val_vector  : -1.56|-1.79|-2.57|-3.32|-3.04|-2.38|-3.16|-3.21\n",
      "\n",
      "epoch:11, train_loss: +0.093, val_loss: -2.555, \n",
      "train_vector: -1.51|-1.79|-2.57|-3.30|-3.04|-2.39|-3.15|-3.24, \n",
      "val_vector  : -0.81|-1.97|-2.56|-3.28|-3.04|-2.39|-3.17|-3.20\n",
      "\n",
      "epoch:12, train_loss: +0.093, val_loss: -2.605, \n",
      "train_vector: -1.50|-1.78|-2.57|-3.32|-3.03|-2.40|-3.16|-3.24, \n",
      "val_vector  : -1.17|-2.02|-2.57|-3.33|-2.97|-2.39|-3.18|-3.21\n",
      "\n",
      "epoch:13, train_loss: +0.090, val_loss: -2.621, \n",
      "train_vector: -1.53|-1.86|-2.57|-3.31|-3.04|-2.40|-3.17|-3.24, \n",
      "val_vector  : -1.29|-2.03|-2.57|-3.24|-3.05|-2.40|-3.17|-3.21\n",
      "\n",
      "epoch:14, train_loss: +0.090, val_loss: -2.672, \n",
      "train_vector: -1.52|-1.88|-2.57|-3.31|-3.05|-2.40|-3.17|-3.24, \n",
      "val_vector  : -1.58|-2.04|-2.57|-3.34|-3.04|-2.40|-3.19|-3.21\n",
      "\n",
      "epoch:15, train_loss: +0.089, val_loss: -2.477, \n",
      "train_vector: -1.52|-1.93|-2.58|-3.32|-3.05|-2.40|-3.17|-3.24, \n",
      "val_vector  : -1.28|-2.05|-2.48|-2.82|-2.94|-2.26|-3.09|-2.89\n",
      "\n",
      "epoch:16, train_loss: +0.089, val_loss: -2.586, \n",
      "train_vector: -1.52|-1.92|-2.58|-3.33|-3.05|-2.40|-3.17|-3.25, \n",
      "val_vector  : -0.96|-1.97|-2.57|-3.34|-3.04|-2.40|-3.20|-3.20\n",
      "\n",
      "epoch:17, train_loss: +0.090, val_loss: -2.666, \n",
      "train_vector: -1.51|-1.91|-2.58|-3.33|-3.04|-2.40|-3.18|-3.25, \n",
      "val_vector  : -1.57|-2.03|-2.56|-3.29|-3.06|-2.40|-3.20|-3.22\n",
      "\n",
      "epoch:18, train_loss: +0.088, val_loss: -2.669, \n",
      "train_vector: -1.52|-1.95|-2.58|-3.34|-3.05|-2.40|-3.18|-3.24, \n",
      "val_vector  : -1.58|-1.99|-2.57|-3.33|-3.07|-2.39|-3.20|-3.22\n",
      "\n",
      "Training completed in 671.2990410327911s\n",
      "-----start bag: 4-----\n",
      "epoch:0, train_loss: +4.100, val_loss: -2.500, \n",
      "train_vector: +0.64|-0.65|-0.92|-1.62|-1.41|-1.26|-1.10|-2.10, \n",
      "val_vector  : -1.43|-1.95|-2.35|-3.17|-2.91|-2.26|-2.89|-3.05\n",
      "\n",
      "epoch:1, train_loss: +0.097, val_loss: -2.581, \n",
      "train_vector: -1.50|-1.89|-2.41|-3.11|-2.93|-2.30|-2.98|-3.11, \n",
      "val_vector  : -1.56|-1.89|-2.48|-3.22|-2.97|-2.35|-3.06|-3.12\n",
      "\n",
      "epoch:2, train_loss: +0.095, val_loss: -2.575, \n",
      "train_vector: -1.47|-1.91|-2.48|-3.22|-2.98|-2.35|-3.06|-3.17, \n",
      "val_vector  : -1.18|-2.07|-2.51|-3.36|-2.96|-2.38|-2.99|-3.16\n",
      "\n",
      "epoch:3, train_loss: +0.093, val_loss: -2.581, \n",
      "train_vector: -1.49|-1.89|-2.51|-3.28|-3.00|-2.37|-3.09|-3.19, \n",
      "val_vector  : -1.30|-1.83|-2.54|-3.16|-3.06|-2.40|-3.13|-3.22\n",
      "\n",
      "epoch:4, train_loss: +0.091, val_loss: -2.677, \n",
      "train_vector: -1.50|-1.93|-2.53|-3.29|-3.01|-2.39|-3.12|-3.20, \n",
      "val_vector  : -1.52|-2.06|-2.56|-3.38|-3.07|-2.41|-3.16|-3.24\n",
      "\n",
      "epoch:5, train_loss: +0.091, val_loss: -2.669, \n",
      "train_vector: -1.49|-1.94|-2.54|-3.31|-3.03|-2.40|-3.13|-3.21, \n",
      "val_vector  : -1.60|-2.01|-2.57|-3.39|-3.07|-2.41|-3.14|-3.16\n",
      "\n",
      "epoch:6, train_loss: +0.090, val_loss: -2.651, \n",
      "train_vector: -1.49|-1.94|-2.55|-3.31|-3.03|-2.40|-3.15|-3.22, \n",
      "val_vector  : -1.61|-2.03|-2.57|-3.26|-2.97|-2.41|-3.15|-3.20\n",
      "\n",
      "epoch:7, train_loss: +0.091, val_loss: -2.645, \n",
      "train_vector: -1.48|-1.94|-2.56|-3.31|-3.03|-2.40|-3.15|-3.22, \n",
      "val_vector  : -1.60|-2.06|-2.57|-3.07|-3.09|-2.42|-3.17|-3.17\n",
      "\n",
      "epoch:8, train_loss: +0.090, val_loss: -2.530, \n",
      "train_vector: -1.49|-1.93|-2.57|-3.33|-3.04|-2.40|-3.16|-3.23, \n",
      "val_vector  : -1.56|-1.97|-2.56|-2.55|-3.01|-2.41|-3.08|-3.11\n",
      "\n",
      "epoch:9, train_loss: +0.090, val_loss: -2.585, \n",
      "train_vector: -1.48|-1.96|-2.57|-3.33|-3.05|-2.41|-3.17|-3.23, \n",
      "val_vector  : -1.16|-2.09|-2.55|-3.28|-2.99|-2.39|-3.09|-3.14\n",
      "\n",
      "epoch:10, train_loss: +0.090, val_loss: -2.670, \n",
      "train_vector: -1.48|-1.95|-2.57|-3.33|-3.05|-2.40|-3.17|-3.23, \n",
      "val_vector  : -1.52|-2.09|-2.60|-3.31|-3.00|-2.42|-3.20|-3.24\n",
      "\n",
      "epoch:11, train_loss: +0.086, val_loss: -2.717, \n",
      "train_vector: -1.56|-2.02|-2.58|-3.37|-3.07|-2.41|-3.18|-3.24, \n",
      "val_vector  : -1.59|-2.11|-2.60|-3.42|-3.10|-2.43|-3.22|-3.26\n",
      "\n",
      "epoch:12, train_loss: +0.085, val_loss: -2.697, \n",
      "train_vector: -1.56|-2.02|-2.58|-3.37|-3.07|-2.41|-3.18|-3.24, \n",
      "val_vector  : -1.57|-2.07|-2.60|-3.38|-3.11|-2.43|-3.18|-3.25\n",
      "\n",
      "epoch:13, train_loss: +0.085, val_loss: -2.716, \n",
      "train_vector: -1.56|-2.03|-2.58|-3.37|-3.07|-2.41|-3.18|-3.24, \n",
      "val_vector  : -1.62|-2.11|-2.60|-3.41|-3.11|-2.43|-3.21|-3.25\n",
      "\n",
      "epoch:14, train_loss: +0.086, val_loss: -2.698, \n",
      "train_vector: -1.55|-2.02|-2.58|-3.37|-3.07|-2.41|-3.18|-3.25, \n",
      "val_vector  : -1.60|-2.04|-2.60|-3.36|-3.08|-2.43|-3.22|-3.26\n",
      "\n",
      "epoch:15, train_loss: +0.085, val_loss: -2.716, \n",
      "train_vector: -1.56|-2.04|-2.58|-3.38|-3.07|-2.41|-3.19|-3.25, \n",
      "val_vector  : -1.62|-2.07|-2.60|-3.42|-3.11|-2.43|-3.21|-3.26\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:16, train_loss: +0.085, val_loss: -2.685, \n",
      "train_vector: -1.56|-2.04|-2.58|-3.37|-3.07|-2.41|-3.19|-3.25, \n",
      "val_vector  : -1.59|-2.02|-2.60|-3.28|-3.08|-2.43|-3.22|-3.26\n",
      "\n",
      "epoch:17, train_loss: +0.085, val_loss: -2.708, \n",
      "train_vector: -1.57|-2.04|-2.58|-3.37|-3.07|-2.41|-3.19|-3.25, \n",
      "val_vector  : -1.52|-2.12|-2.60|-3.43|-3.10|-2.43|-3.23|-3.24\n",
      "\n",
      "epoch:18, train_loss: +0.083, val_loss: -2.720, \n",
      "train_vector: -1.59|-2.08|-2.59|-3.39|-3.08|-2.41|-3.20|-3.26, \n",
      "val_vector  : -1.59|-2.11|-2.60|-3.43|-3.10|-2.43|-3.23|-3.27\n",
      "\n",
      "Training completed in 673.7353270053864s\n",
      "-----start bag: 5-----\n",
      "epoch:0, train_loss: +7.878, val_loss: -2.414, \n",
      "train_vector: +1.61|+2.07|-0.80|-1.42|-1.68|-0.44|-1.34|-1.72, \n",
      "val_vector  : -1.43|-1.71|-2.27|-3.04|-2.87|-2.18|-2.87|-2.95\n",
      "\n",
      "epoch:1, train_loss: +0.117, val_loss: -2.535, \n",
      "train_vector: -1.45|-1.39|-2.41|-2.93|-2.94|-2.24|-2.94|-3.05, \n",
      "val_vector  : -1.37|-1.78|-2.48|-3.25|-2.96|-2.29|-3.02|-3.14\n",
      "\n",
      "epoch:2, train_loss: +0.101, val_loss: -2.568, \n",
      "train_vector: -1.50|-1.68|-2.47|-3.10|-2.98|-2.30|-3.03|-3.14, \n",
      "val_vector  : -1.53|-1.72|-2.50|-3.26|-2.98|-2.33|-3.06|-3.17\n",
      "\n",
      "epoch:3, train_loss: +0.097, val_loss: -2.618, \n",
      "train_vector: -1.51|-1.76|-2.49|-3.15|-3.00|-2.33|-3.08|-3.16, \n",
      "val_vector  : -1.55|-1.96|-2.51|-3.28|-2.99|-2.35|-3.10|-3.20\n",
      "\n",
      "epoch:4, train_loss: +0.095, val_loss: -2.592, \n",
      "train_vector: -1.50|-1.81|-2.51|-3.21|-3.00|-2.35|-3.10|-3.18, \n",
      "val_vector  : -1.40|-1.91|-2.52|-3.24|-3.00|-2.36|-3.10|-3.20\n",
      "\n",
      "epoch:5, train_loss: +0.093, val_loss: -2.629, \n",
      "train_vector: -1.51|-1.86|-2.52|-3.23|-3.02|-2.36|-3.13|-3.19, \n",
      "val_vector  : -1.41|-2.07|-2.54|-3.34|-3.01|-2.37|-3.06|-3.22\n",
      "\n",
      "epoch:6, train_loss: +0.091, val_loss: -2.616, \n",
      "train_vector: -1.53|-1.87|-2.54|-3.28|-3.02|-2.37|-3.14|-3.20, \n",
      "val_vector  : -1.45|-1.93|-2.56|-3.21|-3.02|-2.38|-3.15|-3.23\n",
      "\n",
      "epoch:7, train_loss: +0.092, val_loss: -2.664, \n",
      "train_vector: -1.52|-1.84|-2.55|-3.28|-3.03|-2.38|-3.14|-3.21, \n",
      "val_vector  : -1.56|-2.08|-2.56|-3.35|-3.01|-2.39|-3.13|-3.24\n",
      "\n",
      "epoch:8, train_loss: +0.090, val_loss: -2.669, \n",
      "train_vector: -1.53|-1.90|-2.56|-3.30|-3.04|-2.38|-3.15|-3.21, \n",
      "val_vector  : -1.57|-2.08|-2.57|-3.34|-3.00|-2.40|-3.15|-3.23\n",
      "\n",
      "epoch:9, train_loss: +0.089, val_loss: -2.674, \n",
      "train_vector: -1.54|-1.90|-2.56|-3.31|-3.04|-2.38|-3.16|-3.22, \n",
      "val_vector  : -1.59|-2.03|-2.58|-3.36|-3.03|-2.40|-3.17|-3.24\n",
      "\n",
      "epoch:10, train_loss: +0.089, val_loss: -2.609, \n",
      "train_vector: -1.55|-1.92|-2.57|-3.32|-3.05|-2.39|-3.16|-3.22, \n",
      "val_vector  : -1.60|-1.54|-2.57|-3.37|-2.99|-2.40|-3.16|-3.24\n",
      "\n",
      "epoch:11, train_loss: +0.089, val_loss: -2.671, \n",
      "train_vector: -1.55|-1.89|-2.57|-3.31|-3.05|-2.39|-3.16|-3.23, \n",
      "val_vector  : -1.59|-1.99|-2.57|-3.38|-3.04|-2.40|-3.17|-3.23\n",
      "\n",
      "epoch:12, train_loss: +0.089, val_loss: -2.631, \n",
      "train_vector: -1.55|-1.89|-2.57|-3.32|-3.05|-2.39|-3.17|-3.23, \n",
      "val_vector  : -1.31|-1.94|-2.58|-3.37|-3.02|-2.40|-3.18|-3.25\n",
      "\n",
      "epoch:13, train_loss: +0.088, val_loss: -2.670, \n",
      "train_vector: -1.55|-1.94|-2.57|-3.33|-3.06|-2.39|-3.17|-3.23, \n",
      "val_vector  : -1.45|-2.08|-2.57|-3.38|-3.05|-2.40|-3.18|-3.25\n",
      "\n",
      "epoch:14, train_loss: +0.088, val_loss: -2.668, \n",
      "train_vector: -1.56|-1.93|-2.58|-3.33|-3.06|-2.39|-3.17|-3.23, \n",
      "val_vector  : -1.51|-2.00|-2.59|-3.38|-3.05|-2.40|-3.16|-3.25\n",
      "\n",
      "epoch:15, train_loss: +0.087, val_loss: -2.676, \n",
      "train_vector: -1.56|-1.96|-2.58|-3.33|-3.06|-2.39|-3.17|-3.23, \n",
      "val_vector  : -1.60|-2.09|-2.58|-3.25|-3.04|-2.40|-3.19|-3.25\n",
      "\n",
      "epoch:16, train_loss: +0.087, val_loss: -2.629, \n",
      "train_vector: -1.57|-1.96|-2.58|-3.33|-3.07|-2.39|-3.17|-3.23, \n",
      "val_vector  : -1.19|-2.09|-2.59|-3.26|-3.06|-2.40|-3.19|-3.26\n",
      "\n",
      "epoch:17, train_loss: +0.087, val_loss: -2.650, \n",
      "train_vector: -1.57|-1.94|-2.58|-3.34|-3.07|-2.39|-3.18|-3.23, \n",
      "val_vector  : -1.44|-1.95|-2.59|-3.39|-3.04|-2.41|-3.16|-3.24\n",
      "\n",
      "epoch:18, train_loss: +0.087, val_loss: -2.656, \n",
      "train_vector: -1.56|-1.94|-2.58|-3.34|-3.06|-2.39|-3.18|-3.24, \n",
      "val_vector  : -1.51|-1.99|-2.59|-3.33|-3.05|-2.40|-3.13|-3.24\n",
      "\n",
      "Training completed in 675.7414267063141s\n",
      "-----start bag: 6-----\n",
      "epoch:0, train_loss: +6.615, val_loss: -2.324, \n",
      "train_vector: +1.54|+0.48|-0.55|-1.09|-0.79|-1.19|-1.66|-2.03, \n",
      "val_vector  : -1.39|-1.62|-2.13|-2.90|-2.62|-2.21|-2.83|-2.89\n",
      "\n",
      "epoch:1, train_loss: +0.101, val_loss: -2.569, \n",
      "train_vector: -1.48|-1.81|-2.33|-3.08|-2.93|-2.28|-2.95|-3.07, \n",
      "val_vector  : -1.48|-2.02|-2.41|-3.24|-2.96|-2.32|-3.01|-3.12\n",
      "\n",
      "epoch:2, train_loss: +0.095, val_loss: -2.594, \n",
      "train_vector: -1.48|-1.91|-2.43|-3.19|-3.00|-2.33|-3.05|-3.14, \n",
      "val_vector  : -1.57|-1.80|-2.48|-3.30|-3.01|-2.35|-3.08|-3.16\n",
      "\n",
      "epoch:3, train_loss: +0.092, val_loss: -2.638, \n",
      "train_vector: -1.51|-1.91|-2.48|-3.25|-3.03|-2.36|-3.10|-3.17, \n",
      "val_vector  : -1.57|-2.01|-2.52|-3.32|-3.01|-2.38|-3.11|-3.19\n",
      "\n",
      "epoch:4, train_loss: +0.090, val_loss: -2.613, \n",
      "train_vector: -1.52|-1.94|-2.52|-3.29|-3.04|-2.37|-3.12|-3.19, \n",
      "val_vector  : -1.30|-2.06|-2.54|-3.32|-3.02|-2.38|-3.10|-3.17\n",
      "\n",
      "epoch:5, train_loss: +0.090, val_loss: -2.614, \n",
      "train_vector: -1.51|-1.94|-2.54|-3.31|-3.04|-2.38|-3.14|-3.20, \n",
      "val_vector  : -1.36|-1.98|-2.56|-3.38|-2.97|-2.40|-3.13|-3.14\n",
      "\n",
      "epoch:6, train_loss: +0.088, val_loss: -2.623, \n",
      "train_vector: -1.53|-1.97|-2.56|-3.33|-3.04|-2.39|-3.14|-3.21, \n",
      "val_vector  : -1.46|-1.86|-2.55|-3.36|-3.00|-2.41|-3.15|-3.19\n",
      "\n",
      "epoch:7, train_loss: +0.088, val_loss: -2.615, \n",
      "train_vector: -1.54|-1.97|-2.56|-3.34|-3.06|-2.39|-3.15|-3.22, \n",
      "val_vector  : -1.60|-1.59|-2.58|-3.40|-3.04|-2.41|-3.11|-3.20\n",
      "\n",
      "epoch:8, train_loss: +0.088, val_loss: -2.639, \n",
      "train_vector: -1.53|-1.98|-2.57|-3.34|-3.06|-2.40|-3.15|-3.22, \n",
      "val_vector  : -1.51|-1.82|-2.58|-3.36|-3.04|-2.41|-3.17|-3.22\n",
      "\n",
      "epoch:9, train_loss: +0.087, val_loss: -2.648, \n",
      "train_vector: -1.55|-1.98|-2.57|-3.35|-3.07|-2.40|-3.16|-3.23, \n",
      "val_vector  : -1.31|-2.05|-2.56|-3.41|-3.04|-2.41|-3.17|-3.22\n",
      "\n",
      "epoch:10, train_loss: +0.087, val_loss: -2.642, \n",
      "train_vector: -1.56|-1.98|-2.57|-3.35|-3.06|-2.40|-3.17|-3.23, \n",
      "val_vector  : -1.48|-1.92|-2.56|-3.39|-3.03|-2.41|-3.16|-3.18\n",
      "\n",
      "epoch:11, train_loss: +0.087, val_loss: -2.556, \n",
      "train_vector: -1.55|-1.98|-2.57|-3.36|-3.07|-2.40|-3.18|-3.23, \n",
      "val_vector  : -1.61|-2.05|-2.51|-2.97|-2.96|-2.39|-3.14|-2.82\n",
      "\n",
      "epoch:12, train_loss: +0.086, val_loss: -2.571, \n",
      "train_vector: -1.56|-1.99|-2.58|-3.36|-3.07|-2.40|-3.18|-3.24, \n",
      "val_vector  : -1.40|-1.67|-2.55|-3.32|-3.02|-2.39|-3.09|-3.13\n",
      "\n",
      "epoch:13, train_loss: +0.086, val_loss: -2.572, \n",
      "train_vector: -1.56|-1.98|-2.58|-3.36|-3.08|-2.40|-3.18|-3.24, \n",
      "val_vector  : -0.95|-1.94|-2.57|-3.41|-3.01|-2.41|-3.14|-3.14\n",
      "\n",
      "epoch:14, train_loss: +0.087, val_loss: -2.633, \n",
      "train_vector: -1.56|-1.97|-2.58|-3.37|-3.08|-2.40|-3.18|-3.24, \n",
      "val_vector  : -1.31|-2.06|-2.59|-3.25|-3.05|-2.42|-3.19|-3.20\n",
      "\n",
      "epoch:15, train_loss: +0.086, val_loss: -2.664, \n",
      "train_vector: -1.57|-1.98|-2.58|-3.37|-3.08|-2.40|-3.19|-3.24, \n",
      "val_vector  : -1.60|-1.87|-2.59|-3.38|-3.06|-2.42|-3.16|-3.22\n",
      "\n",
      "epoch:16, train_loss: +0.086, val_loss: -2.681, \n",
      "train_vector: -1.57|-1.97|-2.58|-3.37|-3.08|-2.40|-3.19|-3.24, \n",
      "val_vector  : -1.57|-1.98|-2.60|-3.40|-3.06|-2.42|-3.18|-3.24\n",
      "\n",
      "epoch:17, train_loss: +0.085, val_loss: -2.602, \n",
      "train_vector: -1.58|-2.00|-2.58|-3.38|-3.08|-2.40|-3.19|-3.24, \n",
      "val_vector  : -1.48|-2.07|-2.55|-3.23|-2.96|-2.41|-3.10|-3.02\n",
      "\n",
      "epoch:18, train_loss: +0.086, val_loss: -2.518, \n",
      "train_vector: -1.57|-1.98|-2.58|-3.38|-3.08|-2.40|-3.19|-3.24, \n",
      "val_vector  : -1.35|-1.80|-2.55|-2.94|-2.95|-2.41|-3.12|-3.02\n",
      "\n",
      "Training completed in 671.1987047195435s\n",
      "-----start bag: 7-----\n",
      "epoch:0, train_loss: +5.051, val_loss: -2.418, \n",
      "train_vector: +0.98|+0.30|-1.06|-1.39|-0.97|-0.82|-0.94|-2.27, \n",
      "val_vector  : -1.53|-1.78|-2.36|-3.13|-2.68|-2.05|-2.80|-3.01\n",
      "\n",
      "epoch:1, train_loss: +0.102, val_loss: -2.574, \n",
      "train_vector: -1.50|-1.80|-2.41|-3.19|-2.87|-2.15|-2.90|-3.09, \n",
      "val_vector  : -1.51|-2.05|-2.45|-3.28|-2.98|-2.19|-2.99|-3.14\n",
      "\n",
      "epoch:2, train_loss: +0.097, val_loss: -2.610, \n",
      "train_vector: -1.47|-1.88|-2.47|-3.26|-2.98|-2.22|-3.00|-3.16, \n",
      "val_vector  : -1.52|-2.05|-2.48|-3.33|-3.03|-2.24|-3.04|-3.18\n",
      "\n",
      "epoch:3, train_loss: +0.095, val_loss: -2.611, \n",
      "train_vector: -1.46|-1.90|-2.50|-3.29|-3.02|-2.27|-3.05|-3.19, \n",
      "val_vector  : -1.58|-1.98|-2.51|-3.30|-3.00|-2.28|-3.05|-3.18\n",
      "\n",
      "epoch:4, train_loss: +0.093, val_loss: -2.614, \n",
      "train_vector: -1.46|-1.95|-2.52|-3.30|-3.03|-2.30|-3.08|-3.21, \n",
      "val_vector  : -1.56|-2.01|-2.52|-3.32|-3.00|-2.31|-3.06|-3.12\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:5, train_loss: +0.091, val_loss: -2.616, \n",
      "train_vector: -1.48|-1.98|-2.54|-3.32|-3.04|-2.32|-3.10|-3.22, \n",
      "val_vector  : -1.58|-2.08|-2.53|-3.18|-2.97|-2.34|-3.10|-3.15\n",
      "\n",
      "epoch:6, train_loss: +0.092, val_loss: -2.602, \n",
      "train_vector: -1.44|-1.95|-2.55|-3.33|-3.04|-2.35|-3.13|-3.23, \n",
      "val_vector  : -1.33|-1.81|-2.54|-3.36|-3.05|-2.34|-3.15|-3.22\n",
      "\n",
      "epoch:7, train_loss: +0.091, val_loss: -2.603, \n",
      "train_vector: -1.45|-1.97|-2.56|-3.34|-3.05|-2.37|-3.15|-3.23, \n",
      "val_vector  : -1.46|-1.91|-2.53|-3.25|-3.03|-2.36|-3.11|-3.16\n",
      "\n",
      "epoch:8, train_loss: +0.090, val_loss: -2.651, \n",
      "train_vector: -1.46|-1.98|-2.56|-3.34|-3.06|-2.37|-3.16|-3.23, \n",
      "val_vector  : -1.59|-1.96|-2.56|-3.37|-2.97|-2.37|-3.18|-3.20\n",
      "\n",
      "epoch:9, train_loss: +0.088, val_loss: -2.641, \n",
      "train_vector: -1.51|-1.99|-2.56|-3.35|-3.06|-2.38|-3.17|-3.24, \n",
      "val_vector  : -1.43|-1.90|-2.56|-3.38|-3.07|-2.37|-3.17|-3.23\n",
      "\n",
      "epoch:10, train_loss: +0.088, val_loss: -2.673, \n",
      "train_vector: -1.51|-2.01|-2.57|-3.35|-3.07|-2.38|-3.18|-3.24, \n",
      "val_vector  : -1.52|-2.07|-2.57|-3.38|-3.07|-2.38|-3.18|-3.23\n",
      "\n",
      "epoch:11, train_loss: +0.088, val_loss: -2.649, \n",
      "train_vector: -1.50|-1.99|-2.57|-3.36|-3.07|-2.39|-3.18|-3.24, \n",
      "val_vector  : -1.42|-2.07|-2.57|-3.25|-3.07|-2.38|-3.18|-3.25\n",
      "\n",
      "epoch:12, train_loss: +0.089, val_loss: -2.616, \n",
      "train_vector: -1.50|-1.97|-2.57|-3.35|-3.07|-2.39|-3.19|-3.24, \n",
      "val_vector  : -1.48|-2.08|-2.55|-3.10|-3.02|-2.36|-3.11|-3.21\n",
      "\n",
      "epoch:13, train_loss: +0.088, val_loss: -2.651, \n",
      "train_vector: -1.50|-2.01|-2.57|-3.36|-3.06|-2.39|-3.19|-3.25, \n",
      "val_vector  : -1.56|-2.03|-2.56|-3.35|-3.02|-2.36|-3.15|-3.18\n",
      "\n",
      "epoch:14, train_loss: +0.088, val_loss: -2.678, \n",
      "train_vector: -1.50|-1.99|-2.57|-3.36|-3.07|-2.40|-3.19|-3.24, \n",
      "val_vector  : -1.56|-2.08|-2.57|-3.40|-3.02|-2.39|-3.18|-3.23\n",
      "\n",
      "epoch:15, train_loss: +0.087, val_loss: -2.521, \n",
      "train_vector: -1.53|-2.02|-2.57|-3.37|-3.08|-2.40|-3.19|-3.25, \n",
      "val_vector  : -0.86|-1.67|-2.56|-3.32|-3.00|-2.38|-3.16|-3.21\n",
      "\n",
      "epoch:16, train_loss: +0.087, val_loss: -2.516, \n",
      "train_vector: -1.52|-2.01|-2.58|-3.37|-3.08|-2.40|-3.20|-3.25, \n",
      "val_vector  : -0.76|-2.05|-2.56|-3.25|-2.98|-2.34|-3.04|-3.15\n",
      "\n",
      "epoch:17, train_loss: +0.087, val_loss: -2.585, \n",
      "train_vector: -1.52|-2.02|-2.58|-3.37|-3.07|-2.40|-3.20|-3.25, \n",
      "val_vector  : -0.99|-1.97|-2.57|-3.25|-3.08|-2.38|-3.21|-3.24\n",
      "\n",
      "epoch:18, train_loss: +0.087, val_loss: -2.488, \n",
      "train_vector: -1.52|-2.01|-2.58|-3.37|-3.08|-2.40|-3.20|-3.25, \n",
      "val_vector  : -1.03|-1.79|-2.56|-3.07|-2.94|-2.34|-3.07|-3.12\n",
      "\n",
      "Training completed in 668.247674703598s\n",
      "-----start bag: 8-----\n",
      "epoch:0, train_loss: +4.131, val_loss: -2.484, \n",
      "train_vector: +0.47|-0.10|-0.72|-1.99|-1.54|-1.34|-0.94|-2.00, \n",
      "val_vector  : -1.53|-1.85|-2.31|-3.21|-2.92|-2.26|-2.76|-3.04\n",
      "\n",
      "epoch:1, train_loss: +0.100, val_loss: -2.576, \n",
      "train_vector: -1.50|-1.84|-2.36|-3.20|-2.96|-2.28|-2.84|-3.10, \n",
      "val_vector  : -1.46|-2.05|-2.46|-3.24|-3.01|-2.32|-2.93|-3.13\n",
      "\n",
      "epoch:2, train_loss: +0.096, val_loss: -2.629, \n",
      "train_vector: -1.50|-1.86|-2.46|-3.24|-3.00|-2.33|-2.96|-3.17, \n",
      "val_vector  : -1.45|-2.07|-2.52|-3.33|-3.05|-2.38|-3.03|-3.20\n",
      "\n",
      "epoch:3, train_loss: +0.094, val_loss: -2.632, \n",
      "train_vector: -1.50|-1.88|-2.51|-3.26|-3.02|-2.37|-3.03|-3.20, \n",
      "val_vector  : -1.54|-2.08|-2.54|-3.21|-3.00|-2.40|-3.10|-3.18\n",
      "\n",
      "epoch:4, train_loss: +0.093, val_loss: -2.631, \n",
      "train_vector: -1.50|-1.89|-2.53|-3.26|-3.03|-2.38|-3.07|-3.21, \n",
      "val_vector  : -1.58|-1.92|-2.55|-3.36|-3.01|-2.40|-3.02|-3.22\n",
      "\n",
      "epoch:5, train_loss: +0.090, val_loss: -2.605, \n",
      "train_vector: -1.53|-1.91|-2.55|-3.29|-3.04|-2.39|-3.11|-3.22, \n",
      "val_vector  : -1.34|-1.94|-2.59|-3.15|-3.08|-2.41|-3.08|-3.24\n",
      "\n",
      "epoch:6, train_loss: +0.090, val_loss: -2.667, \n",
      "train_vector: -1.54|-1.90|-2.55|-3.31|-3.05|-2.40|-3.12|-3.23, \n",
      "val_vector  : -1.56|-1.99|-2.59|-3.38|-3.08|-2.41|-3.11|-3.21\n",
      "\n",
      "epoch:7, train_loss: +0.088, val_loss: -2.527, \n",
      "train_vector: -1.56|-1.95|-2.56|-3.30|-3.05|-2.40|-3.14|-3.24, \n",
      "val_vector  : -1.13|-1.55|-2.58|-3.35|-3.05|-2.42|-3.04|-3.10\n",
      "\n",
      "epoch:8, train_loss: +0.089, val_loss: -2.688, \n",
      "train_vector: -1.54|-1.94|-2.57|-3.32|-3.06|-2.40|-3.14|-3.24, \n",
      "val_vector  : -1.60|-2.02|-2.60|-3.38|-3.08|-2.42|-3.17|-3.24\n",
      "\n",
      "epoch:9, train_loss: +0.089, val_loss: -2.627, \n",
      "train_vector: -1.52|-1.95|-2.57|-3.32|-3.06|-2.40|-3.16|-3.24, \n",
      "val_vector  : -1.32|-1.97|-2.57|-3.40|-2.99|-2.41|-3.14|-3.20\n",
      "\n",
      "epoch:10, train_loss: +0.088, val_loss: -2.675, \n",
      "train_vector: -1.55|-1.96|-2.57|-3.34|-3.06|-2.41|-3.16|-3.24, \n",
      "val_vector  : -1.50|-2.10|-2.59|-3.30|-3.05|-2.42|-3.20|-3.24\n",
      "\n",
      "epoch:11, train_loss: +0.088, val_loss: -2.700, \n",
      "train_vector: -1.56|-1.94|-2.57|-3.34|-3.07|-2.41|-3.17|-3.24, \n",
      "val_vector  : -1.54|-2.10|-2.60|-3.42|-3.08|-2.42|-3.19|-3.25\n",
      "\n",
      "epoch:12, train_loss: +0.089, val_loss: -2.691, \n",
      "train_vector: -1.51|-1.95|-2.58|-3.35|-3.07|-2.41|-3.17|-3.25, \n",
      "val_vector  : -1.60|-2.05|-2.61|-3.31|-3.10|-2.42|-3.18|-3.25\n",
      "\n",
      "epoch:13, train_loss: +0.087, val_loss: -2.583, \n",
      "train_vector: -1.55|-1.96|-2.58|-3.35|-3.07|-2.41|-3.17|-3.25, \n",
      "val_vector  : -1.62|-1.68|-2.60|-3.16|-3.07|-2.42|-3.06|-3.05\n",
      "\n",
      "epoch:14, train_loss: +0.086, val_loss: -2.687, \n",
      "train_vector: -1.57|-1.99|-2.58|-3.36|-3.07|-2.41|-3.18|-3.25, \n",
      "val_vector  : -1.50|-2.12|-2.61|-3.39|-3.10|-2.42|-3.16|-3.20\n",
      "\n",
      "epoch:15, train_loss: +0.086, val_loss: -2.695, \n",
      "train_vector: -1.56|-1.99|-2.58|-3.36|-3.07|-2.41|-3.18|-3.25, \n",
      "val_vector  : -1.55|-2.11|-2.59|-3.38|-3.10|-2.42|-3.17|-3.24\n",
      "\n",
      "epoch:16, train_loss: +0.086, val_loss: -2.679, \n",
      "train_vector: -1.56|-1.98|-2.58|-3.35|-3.07|-2.41|-3.18|-3.25, \n",
      "val_vector  : -1.63|-1.98|-2.60|-3.40|-3.10|-2.43|-3.03|-3.26\n",
      "\n",
      "epoch:17, train_loss: +0.087, val_loss: -2.644, \n",
      "train_vector: -1.56|-1.97|-2.58|-3.35|-3.07|-2.41|-3.18|-3.25, \n",
      "val_vector  : -1.55|-1.84|-2.60|-3.21|-3.06|-2.42|-3.21|-3.26\n",
      "\n",
      "epoch:18, train_loss: +0.084, val_loss: -2.719, \n",
      "train_vector: -1.60|-2.03|-2.59|-3.39|-3.09|-2.41|-3.19|-3.26, \n",
      "val_vector  : -1.59|-2.12|-2.61|-3.43|-3.10|-2.42|-3.22|-3.26\n",
      "\n",
      "Training completed in 676.4293456077576s\n",
      "-----start bag: 9-----\n",
      "epoch:0, train_loss: +8.825, val_loss: -2.253, \n",
      "train_vector: +2.17|+0.89|-1.20|-1.03|-1.23|-1.17|-0.47|-1.73, \n",
      "val_vector  : -1.44|-1.44|-2.28|-2.39|-2.68|-2.17|-2.73|-2.89\n",
      "\n",
      "epoch:1, train_loss: +0.110, val_loss: -2.515, \n",
      "train_vector: -1.38|-1.71|-2.37|-2.95|-2.83|-2.19|-2.90|-3.04, \n",
      "val_vector  : -1.34|-2.00|-2.42|-3.11|-2.96|-2.22|-2.97|-3.10\n",
      "\n",
      "epoch:2, train_loss: +0.106, val_loss: -2.519, \n",
      "train_vector: -1.38|-1.69|-2.43|-3.06|-2.93|-2.24|-2.98|-3.12, \n",
      "val_vector  : -1.51|-1.93|-2.46|-2.82|-3.00|-2.28|-3.02|-3.16\n",
      "\n",
      "epoch:3, train_loss: +0.099, val_loss: -2.593, \n",
      "train_vector: -1.41|-1.82|-2.48|-3.16|-2.99|-2.30|-3.02|-3.16, \n",
      "val_vector  : -1.52|-1.91|-2.50|-3.28|-3.00|-2.32|-3.03|-3.18\n",
      "\n",
      "epoch:4, train_loss: +0.098, val_loss: -2.571, \n",
      "train_vector: -1.42|-1.78|-2.51|-3.24|-3.02|-2.33|-3.05|-3.18, \n",
      "val_vector  : -1.33|-1.92|-2.52|-3.10|-3.07|-2.34|-3.08|-3.20\n",
      "\n",
      "epoch:5, train_loss: +0.096, val_loss: -2.580, \n",
      "train_vector: -1.44|-1.83|-2.53|-3.27|-3.03|-2.34|-3.08|-3.20, \n",
      "val_vector  : -1.54|-1.93|-2.51|-3.20|-2.94|-2.32|-3.04|-3.16\n",
      "\n",
      "epoch:6, train_loss: +0.095, val_loss: -2.638, \n",
      "train_vector: -1.43|-1.85|-2.54|-3.30|-3.04|-2.34|-3.10|-3.20, \n",
      "val_vector  : -1.48|-1.95|-2.55|-3.37|-3.06|-2.35|-3.14|-3.21\n",
      "\n",
      "epoch:7, train_loss: +0.094, val_loss: -2.631, \n",
      "train_vector: -1.45|-1.87|-2.54|-3.30|-3.05|-2.35|-3.11|-3.21, \n",
      "val_vector  : -1.53|-2.01|-2.56|-3.13|-3.09|-2.37|-3.14|-3.23\n",
      "\n",
      "epoch:8, train_loss: +0.093, val_loss: -2.620, \n",
      "train_vector: -1.46|-1.89|-2.55|-3.32|-3.05|-2.35|-3.12|-3.22, \n",
      "val_vector  : -1.26|-1.94|-2.56|-3.38|-3.07|-2.37|-3.14|-3.23\n",
      "\n",
      "epoch:9, train_loss: +0.091, val_loss: -2.635, \n",
      "train_vector: -1.48|-1.93|-2.55|-3.33|-3.05|-2.36|-3.13|-3.22, \n",
      "val_vector  : -1.53|-2.00|-2.55|-3.19|-3.09|-2.37|-3.13|-3.23\n",
      "\n",
      "epoch:10, train_loss: +0.091, val_loss: -2.667, \n",
      "train_vector: -1.47|-1.94|-2.56|-3.34|-3.06|-2.36|-3.14|-3.22, \n",
      "val_vector  : -1.56|-2.09|-2.55|-3.30|-3.09|-2.38|-3.17|-3.21\n",
      "\n",
      "epoch:11, train_loss: +0.091, val_loss: -2.484, \n",
      "train_vector: -1.47|-1.93|-2.56|-3.35|-3.06|-2.36|-3.15|-3.23, \n",
      "val_vector  : -1.29|-1.63|-2.56|-2.73|-3.08|-2.35|-3.04|-3.20\n",
      "\n",
      "epoch:12, train_loss: +0.091, val_loss: -2.608, \n",
      "train_vector: -1.49|-1.91|-2.56|-3.35|-3.06|-2.37|-3.15|-3.23, \n",
      "val_vector  : -1.50|-1.83|-2.53|-3.29|-3.03|-2.37|-3.13|-3.19\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:13, train_loss: +0.091, val_loss: -2.602, \n",
      "train_vector: -1.50|-1.87|-2.56|-3.35|-3.06|-2.37|-3.15|-3.23, \n",
      "val_vector  : -1.51|-1.87|-2.56|-3.09|-3.08|-2.39|-3.11|-3.21\n",
      "\n",
      "epoch:14, train_loss: +0.091, val_loss: -2.542, \n",
      "train_vector: -1.49|-1.91|-2.56|-3.35|-3.06|-2.37|-3.15|-3.23, \n",
      "val_vector  : -1.57|-1.85|-2.53|-3.19|-2.91|-2.33|-2.95|-2.99\n",
      "\n",
      "epoch:15, train_loss: +0.091, val_loss: -2.610, \n",
      "train_vector: -1.50|-1.89|-2.57|-3.36|-3.06|-2.37|-3.16|-3.23, \n",
      "val_vector  : -1.48|-2.09|-2.57|-2.86|-3.10|-2.39|-3.16|-3.23\n",
      "\n",
      "epoch:16, train_loss: +0.090, val_loss: -2.596, \n",
      "train_vector: -1.52|-1.90|-2.57|-3.37|-3.07|-2.37|-3.16|-3.23, \n",
      "val_vector  : -1.57|-2.01|-2.56|-3.33|-2.88|-2.38|-3.04|-3.01\n",
      "\n",
      "epoch:17, train_loss: +0.087, val_loss: -2.614, \n",
      "train_vector: -1.54|-2.00|-2.57|-3.39|-3.07|-2.37|-3.17|-3.24, \n",
      "val_vector  : -1.53|-1.96|-2.56|-3.07|-3.09|-2.39|-3.10|-3.22\n",
      "\n",
      "epoch:18, train_loss: +0.086, val_loss: -2.652, \n",
      "train_vector: -1.55|-2.02|-2.57|-3.39|-3.08|-2.38|-3.17|-3.24, \n",
      "val_vector  : -1.44|-2.06|-2.57|-3.40|-3.06|-2.38|-3.11|-3.18\n",
      "\n",
      "Training completed in 635.0127441883087s\n"
     ]
    }
   ],
   "source": [
    "for bag_i in range(10):\n",
    "    print('-----start bag: {}-----'.format(bag_i))\n",
    "    # sample rows\n",
    "    np.random.shuffle(all_data)\n",
    "    \n",
    "    # sample columns\n",
    "    sample_index = torch.rand(dim)>sample_rate\n",
    "    d_bag_i = sample_index.sum().item()\n",
    "    \n",
    "    train_list = [Data(**sample_feature(d,sample_index)) for d in all_data[:70000]]\n",
    "    train_dl = DataLoader(train_list,batch_size,shuffle=True)\n",
    "    val_list = [Data(**sample_feature(d,sample_index)) for d in all_data[70000:]]\n",
    "    val_dl = DataLoader(val_list,batch_size,shuffle=False)   \n",
    "\n",
    "    model = GNN_multiHead_interleave_stacking(reuse,block,head,d_bag_i,layer,factor).to('cuda:0')\n",
    "    paras = trainable_parameter(model)\n",
    "    opt = RAdam(paras,lr=lr)\n",
    "    scheduler = ReduceLROnPlateau(opt, 'min',factor=0.5,patience=5,min_lr=1e-05)\n",
    "    \n",
    "    model,train_loss_perType,val_loss_perType,bestWeight = train_type_earlyStop(opt,model,epochs,train_dl,val_dl,paras,clip,\\\n",
    "                                                                    scheduler=scheduler,logLoss=logLoss,threshold=threshold)\n",
    "    # predict test\n",
    "    for type_i in range(8):\n",
    "        # load val data and type_id\n",
    "        with open(data.format('test').split('pickle')[0][:-1]+'_type_'+str(type_i)+'.pickle', 'rb') as handle:\n",
    "            test_data = pickle.load(handle)\n",
    "        test_list = [Data(**sample_feature(d,sample_index)) for d in test_data]\n",
    "        test_dl = DataLoader(test_list,batch_size,shuffle=False)\n",
    "\n",
    "        with open(data.format('test').split('pickle')[0][:-1]+'_id_type_'+str(type_i)+'.pickle', 'rb') as handle:\n",
    "            test_id = pickle.load(handle)\n",
    "\n",
    "        # load model\n",
    "        model.load_state_dict(bestWeight[type_i])\n",
    "\n",
    "        # predict\n",
    "        model.eval()\n",
    "        yhat_list = []\n",
    "        with torch.no_grad():\n",
    "            for data_torch in test_dl:\n",
    "                data_torch = data_torch.to('cuda:0')\n",
    "                yhat_list.append(model(data_torch,False,typeTrain=True))\n",
    "        yhat = torch.cat(yhat_list).cpu().detach().numpy()        \n",
    "\n",
    "        # join\n",
    "        assert yhat.shape[0]==test_id.shape[0],'yhat and test_id should have same shape'\n",
    "        submit_ = dict(zip(test_id,yhat))\n",
    "        test_df['bag_'+str(bag_i)+'type_'+str(type_i)] = test_df.id.map(submit_)\n",
    "        #test_df['fold'+str(i)+'_type'+str(type_i)] = test_df.id.map(submit_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('../Model/GNN_stacking_0815.pickle', 'wb') as handle:\n",
    "#     pickle.dump(bestWeight, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# with open('../Model/GNN_stacking_0815.pickle', 'rb') as handle:\n",
    "#     bestWeight = pickle.load(handle)\n",
    "\n",
    "# model = GNN_multiHead_interleave_stacking(reuse,block,head,dim,layer,factor,edge_in4).to('cuda:0')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df['scalar_coupling_constant'] = np.nanmean(test_df.iloc[:,5:],1)\n",
    "#test = test[['id','yhat']]\n",
    "test_df[['id','scalar_coupling_constant']].to_csv('../Submission/GNN_stacking_0815_bags',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}