GNN_byType_5fold_0728.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "from torch_geometric.data import Data,DataLoader\n",
    "from functions_refactor import *\n",
    "from pytorch_util import *\n",
    "from torch.optim import Adam\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fixed para\n",
    "batch_size = 32\n",
    "clip = 2\n",
    "lr = 1e-4\n",
    "reuse = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# model parameters\n",
    "block = schnet_block\n",
    "head = cat3Head_type\n",
    "data = '../Data/{}_data_atomInfo.pickle'\n",
    "dim = 512\n",
    "layer1 = 3\n",
    "layer2 = 2\n",
    "factor = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "types = [1,2,4,5,7]\n",
    "epochs = [70] * 5\n",
    "#epochs = [2] * 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = '_'.join([str(i).split('}')[1] if '}' in str(i) else str(i) \\\n",
    "                                        for i in [reuse,block,head,data,dim,layer1,layer2,factor]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('../Data/train.csv')\n",
    "test_df = pd.read_csv('../Data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "start type 0\n",
      "epoch:0, train_loss: +1.010, val_loss: +0.296, \n",
      "train_vector: +1.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.30|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.096, val_loss: +0.064, \n",
      "train_vector: +0.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 125.98496174812317s\n",
      "epoch:0, train_loss: +1.034, val_loss: +0.164, \n",
      "train_vector: +1.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.094, val_loss: -0.047, \n",
      "train_vector: +0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 126.48502922058105s\n",
      "epoch:0, train_loss: +1.025, val_loss: +0.208, \n",
      "train_vector: +1.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.21|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.101, val_loss: +0.020, \n",
      "train_vector: +0.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 127.59845185279846s\n",
      "epoch:0, train_loss: +1.032, val_loss: +0.192, \n",
      "train_vector: +1.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.095, val_loss: -0.056, \n",
      "train_vector: +0.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 125.72738909721375s\n",
      "epoch:0, train_loss: +0.937, val_loss: +0.120, \n",
      "train_vector: +0.94|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.078, val_loss: +0.016, \n",
      "train_vector: +0.08|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 127.50072884559631s\n",
      "\n",
      "start type 1\n",
      "epoch:0, train_loss: +0.962, val_loss: +0.019, \n",
      "train_vector: +0.00|+0.96|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.126, val_loss: -0.337, \n",
      "train_vector: +0.00|-0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.34|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 38.94041204452515s\n",
      "epoch:0, train_loss: +0.969, val_loss: +0.331, \n",
      "train_vector: +0.00|+0.97|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.33|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.105, val_loss: -0.079, \n",
      "train_vector: +0.00|-0.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.08|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 39.03940796852112s\n",
      "epoch:0, train_loss: +1.004, val_loss: -0.116, \n",
      "train_vector: +0.00|+1.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.090, val_loss: -0.342, \n",
      "train_vector: +0.00|-0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.34|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 39.26018309593201s\n",
      "epoch:0, train_loss: +0.951, val_loss: -0.000, \n",
      "train_vector: +0.00|+0.95|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.098, val_loss: -0.253, \n",
      "train_vector: +0.00|-0.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 39.2123122215271s\n",
      "epoch:0, train_loss: +0.999, val_loss: -0.117, \n",
      "train_vector: +0.00|+1.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.143, val_loss: -0.268, \n",
      "train_vector: +0.00|-0.14|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.27|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 39.171813011169434s\n",
      "\n",
      "start type 2\n",
      "epoch:0, train_loss: -0.118, val_loss: -0.451, \n",
      "train_vector: +0.00|+0.00|-0.12|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.45|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.564, val_loss: -0.720, \n",
      "train_vector: +0.00|+0.00|-0.56|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.72|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 123.7176444530487s\n",
      "epoch:0, train_loss: -0.117, val_loss: -0.421, \n",
      "train_vector: +0.00|+0.00|-0.12|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.42|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.568, val_loss: -0.711, \n",
      "train_vector: +0.00|+0.00|-0.57|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.71|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 124.11731290817261s\n",
      "epoch:0, train_loss: -0.149, val_loss: -0.424, \n",
      "train_vector: +0.00|+0.00|-0.15|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.42|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.599, val_loss: -0.706, \n",
      "train_vector: +0.00|+0.00|-0.60|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.71|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 124.04985666275024s\n",
      "epoch:0, train_loss: -0.148, val_loss: -0.458, \n",
      "train_vector: +0.00|+0.00|-0.15|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.46|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.603, val_loss: -0.721, \n",
      "train_vector: +0.00|+0.00|-0.60|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.72|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 123.86986136436462s\n",
      "epoch:0, train_loss: -0.150, val_loss: -0.493, \n",
      "train_vector: +0.00|+0.00|-0.15|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.49|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.605, val_loss: -0.742, \n",
      "train_vector: +0.00|+0.00|-0.60|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.74|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 123.76244044303894s\n",
      "\n",
      "start type 3\n",
      "epoch:0, train_loss: -0.667, val_loss: -1.249, \n",
      "train_vector: +0.00|+0.00|+0.00|-0.67|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.25|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.271, val_loss: -1.566, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.27|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.57|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 120.46678614616394s\n",
      "epoch:0, train_loss: -0.626, val_loss: -1.142, \n",
      "train_vector: +0.00|+0.00|+0.00|-0.63|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.14|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.233, val_loss: -1.401, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.23|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.40|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 120.30336666107178s\n",
      "epoch:0, train_loss: -0.683, val_loss: -1.247, \n",
      "train_vector: +0.00|+0.00|+0.00|-0.68|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.25|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.284, val_loss: -1.353, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.28|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.35|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 123.8503966331482s\n",
      "epoch:0, train_loss: -0.542, val_loss: -1.176, \n",
      "train_vector: +0.00|+0.00|+0.00|-0.54|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.18|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.179, val_loss: -1.126, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.18|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.13|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 121.6346845626831s\n",
      "epoch:0, train_loss: -0.618, val_loss: -1.298, \n",
      "train_vector: +0.00|+0.00|+0.00|-0.62|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.30|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.247, val_loss: -1.547, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.25|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.55|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 121.62020897865295s\n",
      "\n",
      "start type 4\n",
      "epoch:0, train_loss: -0.574, val_loss: -1.106, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-0.57|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.11|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train model\n",
    "for type_i,epoch in zip(types,epochs):\n",
    "    print('\\nstart type '+str(type_i))\n",
    "    # load type data for all folds\n",
    "    folds = []\n",
    "    for f in range(5):\n",
    "        with open(data.format('train').split('pickle')[0][:-1]+'_f'+str(f)+'_type_'+str(type_i)+'.pickle', 'rb') as handle:\n",
    "            folds.append(pickle.load(handle))\n",
    "    folds = [[Data(**d) for d in fold] for fold in folds]\n",
    "    \n",
    "    # test data for type_i\n",
    "    with open(data.format('test').split('pickle')[0][:-1]+'_type_'+str(type_i)+'.pickle', 'rb') as handle:\n",
    "        test_data = pickle.load(handle)\n",
    "    test_list = [Data(**d) for d in test_data]\n",
    "    with open(data.format('test').split('pickle')[0][:-1]+'_id_type_'+str(type_i)+'.pickle', 'rb') as handle:\n",
    "            test_id_test = pickle.load(handle)\n",
    "    \n",
    "    # fold\n",
    "    for i in range(5):\n",
    "        print('\\nstart fold '+str(i))\n",
    "        # parpare data\n",
    "        train_list = []\n",
    "        val_list = []\n",
    "        for j in range(5):\n",
    "            if i == j:\n",
    "                val_list.extend(folds[j])\n",
    "            else:\n",
    "                train_list.extend(folds[j])\n",
    "        train_dl = DataLoader(train_list,batch_size,shuffle=True)\n",
    "        val_dl = DataLoader(val_list,batch_size,shuffle=False)        \n",
    "\n",
    "        # train model\n",
    "        model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "        paras = trainable_parameter(model)\n",
    "        opt = Adam(paras,lr=lr)\n",
    "        scheduler = ReduceLROnPlateau(opt, 'min',factor=0.5,patience=5)\n",
    "        model,train_loss_perType,val_loss_perType = train(opt,model,epoch,train_dl,val_dl,paras,clip,True,scheduler=scheduler)\n",
    "    \n",
    "        # predict oof\n",
    "        with open(data.format('train').split('pickle')[0][:-1]+'_f'+str(i)+'_type_'+str(type_i)+'_id.pickle', 'rb') as handle:\n",
    "            test_id = pickle.load(handle)\n",
    "            \n",
    "        test_dl = DataLoader(val_list,batch_size,shuffle=False)\n",
    "        model.eval()\n",
    "        yhat_list = []\n",
    "        with torch.no_grad():\n",
    "            for data_torch in test_dl:\n",
    "                data_torch = data_torch.to('cuda:0')\n",
    "                yhat_list.append(model(data_torch,False,True))\n",
    "        yhat = torch.cat(yhat_list).cpu().detach().numpy()        \n",
    "        assert yhat.shape[0]==test_id.shape[0],'yhat and test_id should have same shape'\n",
    "        \n",
    "        # join\n",
    "        submit_ = dict(zip(test_id,yhat))\n",
    "        train_df['fold'+str(i)+'_type'+str(type_i)] = train_df.id.map(submit_)\n",
    "        \n",
    "        # predict test\n",
    "        test_dl = DataLoader(test_list,batch_size,shuffle=False)\n",
    "        model.eval()\n",
    "        yhat_list = []\n",
    "        with torch.no_grad():\n",
    "            for data_torch in test_dl:\n",
    "                data_torch = data_torch.to('cuda:0')\n",
    "                yhat_list.append(model(data_torch,False,True))\n",
    "        yhat = torch.cat(yhat_list).cpu().detach().numpy()  \n",
    "        \n",
    "        assert yhat.shape[0]==test_id_test.shape[0],'yhat and test_id should have same shape for test'\n",
    "        # join\n",
    "        submit_ = dict(zip(test_id_test,yhat))\n",
    "        test_df['fold'+str(i)+'_type'+str(type_i)] = test_df.id.map(submit_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['yhat'] = np.nanmean(test.iloc[:,5:],1)\n",
    "#test = test[['id','yhat']]\n",
    "test_df.to_csv('../Data/test_oof_'+prefix,index=False)\n",
    "\n",
    "train['yhat'] = np.nanmean(train.iloc[:,6:],1)\n",
    "#train = train[['id','yhat']]\n",
    "train_df.to_csv('../Data/train_oof_'+prefix,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}