GNN_edge_PCA-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "from torch_geometric.data import Data,DataLoader\n",
    "from functions_refactor import *\n",
    "from pytorch_util import *\n",
    "from torch.optim import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# model parameters\n",
    "reuse = False\n",
    "block = MEGNet_block\n",
    "head = feedforwardHead_Update\n",
    "data = '../Data/{}_data_ACSF_expand_PCA.pickle'\n",
    "batch_size = 32\n",
    "dim = 128\n",
    "epochs = 20\n",
    "clip = 0.4\n",
    "layer1 = 3\n",
    "layer2 = 3\n",
    "factor = 2\n",
    "lr = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### train base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, train_loss: +0.383, val_loss: -0.065, \n",
      "train_vector: +4.26|+1.03|+0.13|-0.39|-0.57|+0.13|-0.60|-0.94, \n",
      "val_vector  : +3.99|-0.03|-0.26|-0.73|-1.09|-0.19|-0.93|-1.28\n",
      "\n",
      "epoch:1, train_loss: -0.266, val_loss: -0.582, \n",
      "train_vector: +2.55|+0.34|-0.33|-0.95|-1.10|-0.25|-1.05|-1.35, \n",
      "val_vector  : +1.15|+0.01|-0.47|-0.97|-1.30|-0.37|-1.22|-1.48\n",
      "\n",
      "epoch:2, train_loss: -0.634, val_loss: -0.808, \n",
      "train_vector: +0.88|+0.14|-0.50|-1.18|-1.29|-0.39|-1.20|-1.53, \n",
      "val_vector  : +0.72|-0.29|-0.64|-1.42|-1.40|-0.49|-1.31|-1.63\n",
      "\n",
      "epoch:3, train_loss: -0.759, val_loss: -0.809, \n",
      "train_vector: +0.80|-0.07|-0.61|-1.33|-1.43|-0.48|-1.31|-1.64, \n",
      "val_vector  : +0.27|+0.68|-0.70|-1.54|-1.44|-0.56|-1.46|-1.71\n",
      "\n",
      "epoch:4, train_loss: -0.836, val_loss: -0.913, \n",
      "train_vector: +0.76|-0.14|-0.69|-1.42|-1.53|-0.54|-1.39|-1.73, \n",
      "val_vector  : +0.84|-0.37|-0.80|-1.69|-1.59|-0.60|-1.31|-1.78\n",
      "\n",
      "epoch:5, train_loss: -0.896, val_loss: -1.036, \n",
      "train_vector: +0.72|-0.20|-0.75|-1.51|-1.59|-0.59|-1.44|-1.80, \n",
      "val_vector  : +0.21|-0.20|-0.85|-1.72|-1.71|-0.67|-1.54|-1.83\n",
      "\n",
      "epoch:6, train_loss: -0.952, val_loss: -1.104, \n",
      "train_vector: +0.72|-0.30|-0.81|-1.58|-1.65|-0.64|-1.50|-1.86, \n",
      "val_vector  : +0.25|-0.48|-0.91|-1.88|-1.64|-0.73|-1.59|-1.86\n",
      "\n",
      "epoch:7, train_loss: -0.996, val_loss: -1.131, \n",
      "train_vector: +0.68|-0.32|-0.86|-1.64|-1.71|-0.67|-1.53|-1.91, \n",
      "val_vector  : +0.14|-0.48|-0.92|-1.74|-1.75|-0.74|-1.61|-1.96\n",
      "\n",
      "epoch:8, train_loss: -1.022, val_loss: -1.124, \n",
      "train_vector: +0.68|-0.35|-0.89|-1.67|-1.74|-0.71|-1.56|-1.94, \n",
      "val_vector  : +0.50|-0.73|-0.97|-1.62|-1.81|-0.78|-1.65|-1.92\n",
      "\n",
      "epoch:9, train_loss: -1.060, val_loss: -1.146, \n",
      "train_vector: +0.65|-0.39|-0.92|-1.71|-1.78|-0.74|-1.60|-1.98, \n",
      "val_vector  : +0.27|-0.41|-1.00|-1.69|-1.87|-0.80|-1.71|-1.97\n",
      "\n",
      "epoch:10, train_loss: -1.102, val_loss: -1.207, \n",
      "train_vector: +0.62|-0.46|-0.96|-1.77|-1.83|-0.77|-1.64|-2.02, \n",
      "val_vector  : +0.11|-0.68|-0.99|-1.63|-1.89|-0.80|-1.76|-2.01\n",
      "\n",
      "epoch:11, train_loss: -1.132, val_loss: -1.249, \n",
      "train_vector: +0.63|-0.51|-0.99|-1.81|-1.85|-0.79|-1.67|-2.06, \n",
      "val_vector  : +0.21|-0.72|-1.06|-1.78|-1.91|-0.86|-1.80|-2.08\n",
      "\n",
      "epoch:12, train_loss: -1.156, val_loss: -1.280, \n",
      "train_vector: +0.61|-0.54|-1.01|-1.83|-1.88|-0.82|-1.68|-2.09, \n",
      "val_vector  : +0.24|-0.73|-1.07|-2.02|-1.94|-0.87|-1.74|-2.11\n",
      "\n",
      "epoch:13, train_loss: -1.184, val_loss: -1.306, \n",
      "train_vector: +0.56|-0.54|-1.04|-1.87|-1.91|-0.84|-1.71|-2.12, \n",
      "val_vector  : +0.17|-0.90|-1.11|-1.79|-1.98|-0.91|-1.82|-2.11\n",
      "\n",
      "epoch:14, train_loss: -1.202, val_loss: -1.334, \n",
      "train_vector: +0.55|-0.56|-1.06|-1.89|-1.93|-0.86|-1.73|-2.14, \n",
      "val_vector  : +0.28|-0.94|-1.14|-2.01|-1.96|-0.91|-1.84|-2.15\n",
      "\n",
      "epoch:15, train_loss: -1.227, val_loss: -1.277, \n",
      "train_vector: +0.52|-0.60|-1.08|-1.91|-1.95|-0.87|-1.75|-2.16, \n",
      "val_vector  : -0.08|-0.52|-1.14|-1.77|-1.92|-0.91|-1.74|-2.14\n",
      "\n",
      "epoch:16, train_loss: -1.253, val_loss: -1.337, \n",
      "train_vector: +0.43|-0.61|-1.10|-1.92|-1.98|-0.89|-1.77|-2.19, \n",
      "val_vector  : +0.05|-0.67|-1.16|-2.08|-1.91|-0.95|-1.82|-2.16\n",
      "\n",
      "epoch:17, train_loss: -1.275, val_loss: -1.345, \n",
      "train_vector: +0.38|-0.62|-1.12|-1.95|-1.99|-0.91|-1.78|-2.21, \n",
      "val_vector  : -0.15|-0.44|-1.17|-2.03|-2.04|-0.94|-1.80|-2.18\n",
      "\n",
      "epoch:18, train_loss: -1.291, val_loss: -1.367, \n",
      "train_vector: +0.33|-0.61|-1.13|-1.96|-2.01|-0.92|-1.80|-2.23, \n",
      "val_vector  : -0.04|-0.92|-1.19|-1.65|-2.03|-0.99|-1.87|-2.23\n",
      "\n",
      "epoch:19, train_loss: -1.321, val_loss: -1.452, \n",
      "train_vector: +0.27|-0.66|-1.15|-2.00|-2.03|-0.94|-1.81|-2.25, \n",
      "val_vector  : -0.13|-0.93|-1.22|-2.19|-2.02|-0.98|-1.94|-2.20\n",
      "\n",
      "Training completed in 1675.0824375152588s\n"
     ]
    }
   ],
   "source": [
    "train_dl,val_dl = get_data(data,batch_size)\n",
    "\n",
    "model = GNN_edgeUpdate(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "\n",
    "paras = trainable_parameter(model)\n",
    "opt = Adam(paras,lr=lr)\n",
    "\n",
    "model,train_loss_list,val_loss_list,bestWeight = train_type(opt,model,epochs,train_dl,val_dl,paras,clip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(train_loss_list,val_loss_list,reuse,block,\\\n",
    "             head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)\n",
    "save_model_type(bestWeight,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Continue training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl,val_dl = get_data(data,batch_size)\n",
    "\n",
    "model = GNN_edgeUpdate(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "\n",
    "paras = trainable_parameter(model)\n",
    "opt = Adam(paras,lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "file_name=\"False_<class 'functions_refactor.MEGNet_block'>_<class 'functions_refactor.feedforwardHead_Update'>__data_ACSF_expand_PCA.pickle_32_128_0.4_3_3_2_20_type_1base.tar\"\n",
    "checkpoint = torch.load('../Model/'+file_name)\n",
    "model.load_state_dict(checkpoint['model_state_dict'])\n",
    "opt.load_state_dict(checkpoint['optimizer_state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, train_loss: -1.234, val_loss: -1.369, \n",
      "train_vector: +0.49|-0.61|-1.08|-1.91|-1.96|-0.88|-1.75|-2.17, \n",
      "val_vector  : -0.10|-0.68|-1.14|-2.05|-2.01|-0.95|-1.86|-2.15\n",
      "\n",
      "epoch:1, train_loss: -1.254, val_loss: -1.345, \n",
      "train_vector: +0.44|-0.62|-1.10|-1.94|-1.98|-0.89|-1.77|-2.19, \n",
      "val_vector  : +0.09|-0.60|-1.16|-2.12|-2.01|-0.94|-1.87|-2.15\n",
      "\n",
      "epoch:2, train_loss: -1.276, val_loss: -1.357, \n",
      "train_vector: +0.38|-0.63|-1.12|-1.95|-2.00|-0.91|-1.79|-2.21, \n",
      "val_vector  : +0.23|-0.93|-1.15|-2.12|-1.98|-0.95|-1.81|-2.16\n",
      "\n",
      "epoch:3, train_loss: -1.303, val_loss: -1.269, \n",
      "train_vector: +0.32|-0.66|-1.13|-1.97|-2.02|-0.92|-1.81|-2.23, \n",
      "val_vector  : +0.45|-0.49|-1.18|-2.06|-1.89|-0.96|-1.86|-2.17\n",
      "\n",
      "epoch:4, train_loss: -1.317, val_loss: -1.378, \n",
      "train_vector: +0.29|-0.65|-1.15|-1.98|-2.04|-0.94|-1.82|-2.25, \n",
      "val_vector  : -0.12|-0.78|-1.20|-2.18|-1.91|-0.95|-1.73|-2.15\n",
      "\n",
      "epoch:5, train_loss: -1.335, val_loss: -1.214, \n",
      "train_vector: +0.25|-0.69|-1.16|-1.97|-2.06|-0.95|-1.84|-2.27, \n",
      "val_vector  : +0.53|-0.43|-1.17|-2.14|-1.73|-0.98|-1.62|-2.16\n",
      "\n",
      "epoch:6, train_loss: -1.363, val_loss: -1.346, \n",
      "train_vector: +0.19|-0.70|-1.18|-2.03|-2.08|-0.97|-1.85|-2.29, \n",
      "val_vector  : +0.08|-0.18|-1.23|-2.23|-2.06|-1.02|-1.91|-2.22\n",
      "\n",
      "epoch:7, train_loss: -1.380, val_loss: -1.414, \n",
      "train_vector: +0.15|-0.72|-1.20|-2.03|-2.09|-0.98|-1.87|-2.30, \n",
      "val_vector  : -0.07|-0.88|-1.21|-2.01|-2.04|-1.01|-1.88|-2.22\n",
      "\n",
      "epoch:8, train_loss: -1.394, val_loss: -1.447, \n",
      "train_vector: +0.13|-0.73|-1.21|-2.04|-2.11|-0.99|-1.88|-2.32, \n",
      "val_vector  : -0.25|-1.00|-1.28|-1.60|-2.12|-1.04|-2.00|-2.29\n",
      "\n",
      "epoch:9, train_loss: -1.409, val_loss: -1.485, \n",
      "train_vector: +0.11|-0.75|-1.22|-2.04|-2.13|-1.00|-1.90|-2.34, \n",
      "val_vector  : -0.29|-0.99|-1.26|-2.00|-2.05|-1.05|-1.96|-2.28\n",
      "\n",
      "epoch:10, train_loss: -1.424, val_loss: -1.322, \n",
      "train_vector: +0.07|-0.74|-1.23|-2.08|-2.13|-1.02|-1.91|-2.35, \n",
      "val_vector  : +0.57|-0.36|-1.27|-2.22|-2.08|-1.05|-1.89|-2.27\n",
      "\n",
      "epoch:11, train_loss: -1.450, val_loss: -1.418, \n",
      "train_vector: +0.02|-0.80|-1.25|-2.10|-2.16|-1.03|-1.92|-2.36, \n",
      "val_vector  : -0.15|-0.35|-1.30|-2.13|-2.11|-1.08|-1.98|-2.25\n",
      "\n",
      "epoch:12, train_loss: -1.464, val_loss: -1.458, \n",
      "train_vector: +0.00|-0.80|-1.26|-2.11|-2.17|-1.04|-1.94|-2.39, \n",
      "val_vector  : +0.11|-1.14|-1.30|-1.82|-2.19|-1.07|-1.99|-2.26\n",
      "\n",
      "epoch:13, train_loss: -1.472, val_loss: -1.553, \n",
      "train_vector: -0.02|-0.80|-1.27|-2.11|-2.18|-1.05|-1.95|-2.40, \n",
      "val_vector  : -0.30|-1.04|-1.31|-2.29|-2.14|-1.10|-1.95|-2.29\n",
      "\n",
      "epoch:14, train_loss: -1.493, val_loss: -1.542, \n",
      "train_vector: -0.04|-0.85|-1.28|-2.14|-2.19|-1.06|-1.97|-2.41, \n",
      "val_vector  : -0.22|-1.06|-1.32|-2.21|-2.10|-1.10|-1.99|-2.33\n",
      "\n",
      "epoch:15, train_loss: -1.502, val_loss: -1.422, \n",
      "train_vector: -0.05|-0.84|-1.29|-2.15|-2.21|-1.07|-1.98|-2.43, \n",
      "val_vector  : -0.01|-0.26|-1.33|-2.27|-2.12|-1.10|-1.94|-2.34\n",
      "\n",
      "epoch:16, train_loss: -1.516, val_loss: -1.557, \n",
      "train_vector: -0.07|-0.87|-1.30|-2.15|-2.23|-1.07|-1.99|-2.44, \n",
      "val_vector  : -0.17|-0.98|-1.35|-2.29|-2.20|-1.10|-2.03|-2.34\n",
      "\n",
      "epoch:17, train_loss: -1.525, val_loss: -1.520, \n",
      "train_vector: -0.09|-0.86|-1.31|-2.17|-2.23|-1.08|-1.99|-2.45, \n",
      "val_vector  : +0.08|-0.96|-1.36|-2.25|-2.17|-1.12|-2.06|-2.32\n",
      "\n",
      "epoch:18, train_loss: -1.535, val_loss: -1.526, \n",
      "train_vector: -0.11|-0.86|-1.32|-2.18|-2.25|-1.09|-2.00|-2.46, \n",
      "val_vector  : -0.22|-0.82|-1.36|-2.05|-2.16|-1.13|-2.10|-2.37\n",
      "\n",
      "epoch:19, train_loss: -1.547, val_loss: -1.576, \n",
      "train_vector: -0.13|-0.88|-1.33|-2.18|-2.26|-1.10|-2.02|-2.48, \n",
      "val_vector  : -0.32|-1.03|-1.35|-2.16|-2.21|-1.13|-2.06|-2.35\n",
      "\n",
      "epoch:20, train_loss: -1.554, val_loss: -1.599, \n",
      "train_vector: -0.14|-0.88|-1.34|-2.18|-2.28|-1.11|-2.03|-2.48, \n",
      "val_vector  : -0.36|-0.96|-1.38|-2.31|-2.18|-1.15|-2.08|-2.37\n",
      "\n",
      "epoch:21, train_loss: -1.565, val_loss: -1.555, \n",
      "train_vector: -0.13|-0.90|-1.35|-2.20|-2.29|-1.12|-2.04|-2.50, \n",
      "val_vector  : +0.19|-0.98|-1.39|-2.36|-2.23|-1.14|-2.12|-2.41\n",
      "\n",
      "epoch:22, train_loss: -1.586, val_loss: -1.554, \n",
      "train_vector: -0.18|-0.96|-1.36|-2.19|-2.30|-1.13|-2.05|-2.51, \n",
      "val_vector  : +0.16|-1.15|-1.35|-2.21|-2.25|-1.16|-2.11|-2.35\n",
      "\n",
      "epoch:23, train_loss: -1.591, val_loss: -1.571, \n",
      "train_vector: -0.18|-0.94|-1.36|-2.21|-2.31|-1.13|-2.06|-2.53, \n",
      "val_vector  : -0.22|-0.89|-1.40|-2.19|-2.21|-1.17|-2.10|-2.38\n",
      "\n",
      "epoch:24, train_loss: -1.601, val_loss: -1.597, \n",
      "train_vector: -0.18|-0.97|-1.37|-2.23|-2.32|-1.14|-2.07|-2.53, \n",
      "val_vector  : -0.40|-0.73|-1.42|-2.33|-2.20|-1.17|-2.15|-2.37\n",
      "\n",
      "epoch:25, train_loss: -1.612, val_loss: -1.652, \n",
      "train_vector: -0.20|-0.96|-1.38|-2.24|-2.33|-1.15|-2.08|-2.55, \n",
      "val_vector  : -0.46|-1.23|-1.41|-2.36|-2.19|-1.15|-2.06|-2.36\n",
      "\n",
      "epoch:26, train_loss: -1.619, val_loss: -1.622, \n",
      "train_vector: -0.21|-0.97|-1.39|-2.24|-2.34|-1.16|-2.09|-2.55, \n",
      "val_vector  : -0.26|-1.16|-1.39|-2.21|-2.32|-1.18|-2.08|-2.38\n",
      "\n",
      "epoch:27, train_loss: -1.632, val_loss: -1.596, \n",
      "train_vector: -0.22|-1.00|-1.40|-2.25|-2.36|-1.17|-2.10|-2.57, \n",
      "val_vector  : -0.33|-0.94|-1.41|-2.23|-2.13|-1.19|-2.13|-2.39\n",
      "\n",
      "epoch:28, train_loss: -1.641, val_loss: -1.505, \n",
      "train_vector: -0.24|-1.02|-1.40|-2.26|-2.36|-1.17|-2.10|-2.57, \n",
      "val_vector  : +0.32|-0.60|-1.41|-2.26|-2.30|-1.19|-2.16|-2.45\n",
      "\n",
      "epoch:29, train_loss: -1.649, val_loss: -1.571, \n",
      "train_vector: -0.24|-1.01|-1.41|-2.29|-2.37|-1.18|-2.11|-2.58, \n",
      "val_vector  : +0.39|-1.19|-1.43|-2.41|-2.18|-1.20|-2.13|-2.43\n",
      "\n",
      "epoch:30, train_loss: -1.647, val_loss: -1.596, \n",
      "train_vector: -0.23|-0.98|-1.42|-2.29|-2.37|-1.18|-2.12|-2.59, \n",
      "val_vector  : -0.01|-0.83|-1.44|-2.39|-2.29|-1.20|-2.17|-2.43\n",
      "\n",
      "epoch:31, train_loss: -1.654, val_loss: -1.560, \n",
      "train_vector: -0.24|-1.00|-1.42|-2.27|-2.38|-1.19|-2.12|-2.60, \n",
      "val_vector  : -0.44|-0.48|-1.44|-2.15|-2.27|-1.20|-2.05|-2.45\n",
      "\n",
      "epoch:32, train_loss: -1.667, val_loss: -1.460, \n",
      "train_vector: -0.26|-1.02|-1.43|-2.29|-2.40|-1.19|-2.14|-2.61, \n",
      "val_vector  : +0.41|-0.51|-1.43|-2.01|-2.31|-1.21|-2.18|-2.44\n",
      "\n",
      "epoch:33, train_loss: -1.671, val_loss: -1.552, \n",
      "train_vector: -0.26|-1.02|-1.44|-2.29|-2.40|-1.20|-2.14|-2.62, \n",
      "val_vector  : -0.23|-0.16|-1.47|-2.43|-2.28|-1.21|-2.22|-2.42\n",
      "\n",
      "epoch:34, train_loss: -1.683, val_loss: -1.719, \n",
      "train_vector: -0.27|-1.05|-1.45|-2.30|-2.41|-1.20|-2.15|-2.63, \n",
      "val_vector  : -0.46|-1.31|-1.48|-2.40|-2.24|-1.24|-2.19|-2.44\n",
      "\n",
      "epoch:35, train_loss: -1.684, val_loss: -1.688, \n",
      "train_vector: -0.28|-1.03|-1.45|-2.30|-2.41|-1.21|-2.15|-2.63, \n",
      "val_vector  : -0.30|-1.11|-1.45|-2.44|-2.29|-1.23|-2.21|-2.47\n",
      "\n",
      "epoch:36, train_loss: -1.695, val_loss: -1.537, \n",
      "train_vector: -0.30|-1.07|-1.46|-2.30|-2.42|-1.22|-2.16|-2.64, \n",
      "val_vector  : +0.23|-0.81|-1.47|-2.10|-2.32|-1.22|-2.14|-2.48\n",
      "\n",
      "epoch:37, train_loss: -1.709, val_loss: -1.743, \n",
      "train_vector: -0.32|-1.09|-1.46|-2.34|-2.43|-1.22|-2.16|-2.65, \n",
      "val_vector  : -0.44|-1.29|-1.48|-2.43|-2.33|-1.24|-2.23|-2.49\n",
      "\n",
      "epoch:38, train_loss: -1.706, val_loss: -1.575, \n",
      "train_vector: -0.30|-1.06|-1.47|-2.32|-2.44|-1.23|-2.17|-2.66, \n",
      "val_vector  : -0.51|-0.40|-1.48|-2.25|-2.20|-1.25|-2.07|-2.44\n",
      "\n",
      "epoch:39, train_loss: -1.713, val_loss: -1.666, \n",
      "train_vector: -0.30|-1.08|-1.47|-2.32|-2.44|-1.23|-2.18|-2.67, \n",
      "val_vector  : -0.55|-0.77|-1.48|-2.45|-2.35|-1.22|-1.99|-2.51\n",
      "\n",
      "Training completed in 3320.475394010544s\n"
     ]
    }
   ],
   "source": [
    "model,train_loss_list,val_loss_list,bestWeight = train_type(opt,model,40,train_dl,val_dl,paras,clip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for para in opt.param_groups:\n",
    "    para['lr'] = lr/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, train_loss: -1.852, val_loss: -1.839, \n",
      "train_vector: -0.44|-1.31|-1.55|-2.49|-2.63|-1.28|-2.29|-2.82, \n",
      "val_vector  : -0.61|-1.31|-1.56|-2.56|-2.47|-1.30|-2.30|-2.59\n",
      "\n",
      "epoch:1, train_loss: -1.887, val_loss: -1.792, \n",
      "train_vector: -0.47|-1.38|-1.56|-2.53|-2.68|-1.30|-2.32|-2.87, \n",
      "val_vector  : -0.41|-1.10|-1.56|-2.60|-2.45|-1.31|-2.29|-2.62\n",
      "\n",
      "epoch:2, train_loss: -1.896, val_loss: -1.871, \n",
      "train_vector: -0.48|-1.38|-1.57|-2.54|-2.69|-1.31|-2.33|-2.88, \n",
      "val_vector  : -0.65|-1.37|-1.58|-2.59|-2.52|-1.32|-2.35|-2.61\n",
      "\n",
      "epoch:3, train_loss: -1.910, val_loss: -1.764, \n",
      "train_vector: -0.50|-1.41|-1.58|-2.55|-2.70|-1.31|-2.34|-2.90, \n",
      "val_vector  : -0.37|-0.90|-1.56|-2.54|-2.49|-1.31|-2.31|-2.62\n",
      "\n",
      "epoch:4, train_loss: -1.918, val_loss: -1.818, \n",
      "train_vector: -0.50|-1.42|-1.58|-2.56|-2.72|-1.31|-2.34|-2.91, \n",
      "val_vector  : -0.61|-1.35|-1.58|-2.25|-2.49|-1.32|-2.33|-2.63\n",
      "\n",
      "epoch:5, train_loss: -1.926, val_loss: -1.841, \n",
      "train_vector: -0.51|-1.44|-1.58|-2.56|-2.73|-1.32|-2.35|-2.92, \n",
      "val_vector  : -0.57|-1.51|-1.58|-2.31|-2.48|-1.32|-2.32|-2.62\n",
      "\n",
      "epoch:6, train_loss: -1.929, val_loss: -1.772, \n",
      "train_vector: -0.51|-1.44|-1.59|-2.55|-2.73|-1.32|-2.35|-2.93, \n",
      "val_vector  : -0.24|-1.05|-1.59|-2.49|-2.49|-1.32|-2.34|-2.65\n",
      "\n",
      "epoch:7, train_loss: -1.935, val_loss: -1.827, \n",
      "train_vector: -0.51|-1.45|-1.59|-2.57|-2.75|-1.32|-2.36|-2.93, \n",
      "val_vector  : -0.40|-1.20|-1.59|-2.61|-2.52|-1.32|-2.33|-2.63\n",
      "\n",
      "epoch:8, train_loss: -1.939, val_loss: -1.800, \n",
      "train_vector: -0.53|-1.44|-1.60|-2.57|-2.75|-1.33|-2.36|-2.94, \n",
      "val_vector  : -0.41|-1.08|-1.57|-2.55|-2.52|-1.32|-2.30|-2.64\n",
      "\n",
      "epoch:9, train_loss: -1.946, val_loss: -1.852, \n",
      "train_vector: -0.53|-1.46|-1.60|-2.58|-2.75|-1.33|-2.37|-2.95, \n",
      "val_vector  : -0.52|-1.33|-1.59|-2.54|-2.54|-1.33|-2.33|-2.64\n",
      "\n",
      "epoch:10, train_loss: -1.949, val_loss: -1.744, \n",
      "train_vector: -0.53|-1.45|-1.60|-2.58|-2.77|-1.33|-2.37|-2.95, \n",
      "val_vector  : +0.03|-1.03|-1.57|-2.65|-2.51|-1.33|-2.26|-2.63\n",
      "\n",
      "epoch:11, train_loss: -1.954, val_loss: -1.739, \n",
      "train_vector: -0.55|-1.47|-1.61|-2.59|-2.76|-1.33|-2.37|-2.95, \n",
      "val_vector  : -0.02|-0.84|-1.60|-2.64|-2.53|-1.33|-2.33|-2.63\n",
      "\n",
      "epoch:12, train_loss: -1.959, val_loss: -1.745, \n",
      "train_vector: -0.54|-1.47|-1.61|-2.60|-2.77|-1.34|-2.38|-2.96, \n",
      "val_vector  : +0.12|-1.24|-1.58|-2.41|-2.50|-1.33|-2.36|-2.65\n",
      "\n",
      "epoch:13, train_loss: -1.961, val_loss: -1.873, \n",
      "train_vector: -0.54|-1.47|-1.61|-2.60|-2.78|-1.34|-2.38|-2.97, \n",
      "val_vector  : -0.66|-1.39|-1.59|-2.63|-2.45|-1.33|-2.28|-2.66\n",
      "\n",
      "epoch:14, train_loss: -1.965, val_loss: -1.789, \n",
      "train_vector: -0.54|-1.48|-1.62|-2.60|-2.78|-1.34|-2.39|-2.97, \n",
      "val_vector  : -0.31|-0.87|-1.60|-2.64|-2.51|-1.34|-2.37|-2.66\n",
      "\n",
      "epoch:15, train_loss: -1.971, val_loss: -1.840, \n",
      "train_vector: -0.55|-1.48|-1.62|-2.60|-2.79|-1.35|-2.39|-2.98, \n",
      "val_vector  : -0.67|-1.04|-1.59|-2.49|-2.57|-1.34|-2.36|-2.66\n",
      "\n",
      "epoch:16, train_loss: -1.971, val_loss: -1.816, \n",
      "train_vector: -0.55|-1.49|-1.62|-2.60|-2.79|-1.35|-2.39|-2.98, \n",
      "val_vector  : -0.66|-0.85|-1.59|-2.51|-2.55|-1.34|-2.38|-2.67\n",
      "\n",
      "epoch:17, train_loss: -1.978, val_loss: -1.903, \n",
      "train_vector: -0.56|-1.49|-1.63|-2.61|-2.80|-1.35|-2.40|-2.99, \n",
      "val_vector  : -0.69|-1.42|-1.61|-2.60|-2.51|-1.35|-2.39|-2.66\n",
      "\n",
      "epoch:18, train_loss: -1.976, val_loss: -1.828, \n",
      "train_vector: -0.55|-1.48|-1.63|-2.60|-2.80|-1.35|-2.40|-2.99, \n",
      "val_vector  : -0.66|-1.09|-1.58|-2.44|-2.48|-1.34|-2.38|-2.66\n",
      "\n",
      "epoch:19, train_loss: -1.986, val_loss: -1.856, \n",
      "train_vector: -0.58|-1.50|-1.63|-2.61|-2.80|-1.36|-2.41|-3.00, \n",
      "val_vector  : -0.29|-1.34|-1.61|-2.69|-2.50|-1.35|-2.39|-2.68\n",
      "\n",
      "Training completed in 1657.5069274902344s\n"
     ]
    }
   ],
   "source": [
    "model,train_loss_list,val_loss_list,bestWeight = train_type(opt,model,20,train_dl,val_dl,paras,clip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(train_loss_list,val_loss_list,reuse,block,\\\n",
    "             head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)\n",
    "save_model_type(bestWeight,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### make submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_submission(reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,postStr='base'):\n",
    "    # set up\n",
    "    model = GNN_edgeUpdate(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "    submission = pd.read_csv('../Data/sample_submission.csv')\n",
    "    \n",
    "    for i in range(8):\n",
    "        # load test data and type_id\n",
    "        with open(data.format('test').split('pickle')[0][:-1]+'_type_'+str(i)+'.pickle', 'rb') as handle:\n",
    "            test_data = pickle.load(handle)\n",
    "        test_list = [Data(**d) for d in test_data]\n",
    "        test_dl = DataLoader(test_list,batch_size,shuffle=False)\n",
    "        with open(data.format('test').split('pickle')[0][:-1]+'_id_type_'+str(i)+'.pickle', 'rb') as handle:\n",
    "            test_id = pickle.load(handle)\n",
    "    \n",
    "    \n",
    "        # load model\n",
    "        checkpoint = torch.load('../Model/{}.tar'.format('_'.join([str(i).split('}')[1] if '}' in str(i) else str(i) \\\n",
    "                                            for i in [reuse,block,head,data,batch_size,dim,clip,\\\n",
    "                                                  layer1,layer2,factor,epochs,'type_'+str(i)+postStr]])))\n",
    "        model.load_state_dict(checkpoint['model_state_dict'])\n",
    "    \n",
    "    \n",
    "        # predict\n",
    "        model.eval()\n",
    "        yhat_list = []\n",
    "        with torch.no_grad():\n",
    "            for data_torch in test_dl:\n",
    "                data_torch = data_torch.to('cuda:0')\n",
    "                yhat_list.append(model(data_torch,False,True))\n",
    "        yhat = torch.cat(yhat_list).cpu().detach().numpy()        \n",
    "        \n",
    "        # join\n",
    "        submit_ = dict(zip(test_id,yhat))\n",
    "        submission['type_'+str(i)] = submission.id.map(submit_)\n",
    "    \n",
    "    # save types results    \n",
    "    submission.to_csv('../Submission/{}.csv'.format('_'.join([str(i).split('}')[1] if '}' in str(i) else str(i) \\\n",
    "                                        for i in [reuse,block,head,data,batch_size,dim,clip,\\\n",
    "                                              layer1,layer2,factor,epochs,'all_types'+postStr]])),\\\n",
    "                      index=False)\n",
    "    \n",
    "    # save final results for submission\n",
    "    submission['scalar_coupling_constant'] = submission.iloc[:,2:].mean(1)\n",
    "    submission = submission[['id','scalar_coupling_constant']]\n",
    "    \n",
    "    submission.to_csv('../Submission/{}.csv'.format('_'.join([str(i).split('}')[1] if '}' in str(i) else str(i) \\\n",
    "                                        for i in [reuse,block,head,data,batch_size,dim,clip,\\\n",
    "                                              layer1,layer2,factor,epochs,'final'+postStr]])),\\\n",
    "                      index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission(reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}