Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e9a2d44c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functions_pt import *\n",
    "import pickle\n",
    "from torch.optim import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c359764b",
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_size = 128\n",
    "dropout = 0.1\n",
    "multiple_factor = 1\n",
    "layers = 2\n",
    "batch_size = 64\n",
    "agg = 'mean' # mean,sum,min,max\n",
    "lr = 1e-4\n",
    "clip = 1\n",
    "epochs = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45745666",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9b576db1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "ys = np.load('ys.npy')\n",
    "weights = np.load('weights.npy')\n",
    "length = np.load('length.npy')\n",
    "words_embed = np.load('possible_words_embed.npy')\n",
    "with open('indexes.pkl', 'rb') as f:\n",
    "    indexes = pickle.load( f)\n",
    "\n",
    "# train/val split\n",
    "np.random.seed(0)\n",
    "train_idx = np.random.rand(ys.shape[0])>0.2\n",
    "val_idx = np.logical_not(train_idx)\n",
    "\n",
    "ys_train, ys_val = ys[train_idx], ys[val_idx]\n",
    "weights_train, weights_val = weights[train_idx], weights[val_idx]\n",
    "length_train, length_val = length[train_idx], length[val_idx]\n",
    "indexes_train = [idx for idx,bol in zip(indexes,train_idx) if bol]\n",
    "indexes_val = [idx for idx,bol in zip(indexes,train_idx) if not bol]\n",
    "\n",
    "# set up dataloader\n",
    "dataset_train = CustomDataset(ys_train,length_train,indexes_train,words_embed)\n",
    "sampler = WeightedRandomSampler(weights_train, 50000)\n",
    "dataset_train = DataLoader(dataset_train, batch_size=batch_size, sampler=sampler,collate_fn=collate)\n",
    "\n",
    "dataset_val = CustomDataset(ys_val,length_val,indexes_val,words_embed)\n",
    "sampler = WeightedRandomSampler(weights_val, 10000)\n",
    "dataset_val = DataLoader(dataset_val, batch_size=batch_size, sampler=sampler,collate_fn=collate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a580a586",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "542b5ec9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "03b8612d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = pointNet(layers,embed_size,agg,dropout,multiple_factor).to('cuda')\n",
    "opt = Adam([    {'params': [p for p in model.parameters() if p is not model.w]},\\\n",
    "                {'params': model.w, 'lr': lr/4}\n",
    "            ],lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "047e2e60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, train_loss: +1.361, val_loss: +0.735 \n",
      "\n",
      "epoch:1, train_loss: +1.005, val_loss: +0.571 \n",
      "\n",
      "epoch:2, train_loss: +0.738, val_loss: +0.424 \n",
      "\n",
      "epoch:3, train_loss: +0.534, val_loss: +0.274 \n",
      "\n",
      "epoch:4, train_loss: +0.384, val_loss: +0.242 \n",
      "\n",
      "epoch:5, train_loss: +0.279, val_loss: +0.148 \n",
      "\n",
      "epoch:6, train_loss: +0.204, val_loss: +0.122 \n",
      "\n",
      "epoch:7, train_loss: +0.148, val_loss: +0.092 \n",
      "\n",
      "epoch:8, train_loss: +0.104, val_loss: +0.054 \n",
      "\n",
      "epoch:9, train_loss: +0.069, val_loss: +0.041 \n",
      "\n",
      "epoch:10, train_loss: +0.043, val_loss: +0.028 \n",
      "\n",
      "epoch:11, train_loss: +0.025, val_loss: +0.013 \n",
      "\n",
      "epoch:12, train_loss: +0.014, val_loss: +0.005 \n",
      "\n",
      "epoch:13, train_loss: +0.008, val_loss: +0.004 \n",
      "\n",
      "epoch:14, train_loss: +0.006, val_loss: +0.003 \n",
      "\n",
      "Training completed in 240.26040315628052s\n"
     ]
    }
   ],
   "source": [
    "model,_ = train(opt,model,epochs,dataset_train,dataset_val,model.parameters(),clip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1757039e",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'baseline0.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2ce26f29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor(0.2472, device='cuda:0', requires_grad=True)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f106886",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bc4b1341",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, train_loss: +0.207, val_loss: +0.197 \n",
      "\n",
      "epoch:1, train_loss: +0.187, val_loss: +0.194 \n",
      "\n",
      "epoch:2, train_loss: +0.185, val_loss: +0.193 \n",
      "\n",
      "epoch:3, train_loss: +0.173, val_loss: +0.136 \n",
      "\n",
      "epoch:4, train_loss: +0.085, val_loss: +0.174 \n",
      "\n",
      "epoch:5, train_loss: +0.065, val_loss: +0.139 \n",
      "\n",
      "epoch:6, train_loss: +0.066, val_loss: +0.137 \n",
      "\n",
      "epoch:7, train_loss: +0.067, val_loss: +0.115 \n",
      "\n",
      "epoch:8, train_loss: +0.062, val_loss: +0.105 \n",
      "\n",
      "epoch:9, train_loss: +0.061, val_loss: +0.097 \n",
      "\n",
      "epoch:10, train_loss: +0.062, val_loss: +0.165 \n",
      "\n",
      "epoch:11, train_loss: +0.076, val_loss: +0.135 \n",
      "\n",
      "epoch:12, train_loss: +0.071, val_loss: +0.117 \n",
      "\n",
      "epoch:13, train_loss: +0.061, val_loss: +0.149 \n",
      "\n",
      "epoch:14, train_loss: +0.059, val_loss: +0.092 \n",
      "\n",
      "Training completed in 235.402446269989s\n"
     ]
    }
   ],
   "source": [
    "# floor at 1\n",
    "# out = torch.maximum(self.min_,out.squeeze() + self.w * torch.log2(length))\n",
    "model,_ = train(opt,model,epochs,dataset_train,dataset_val,model.parameters(),clip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f764ccd5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8b3b5512",
   "metadata": {},
   "outputs": [],
   "source": [
    "w = dataset_val.dataset.words_embed\n",
    "l = torch.tensor([2309.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8c5fdb68",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_40386/3274103529.py:1: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  model((torch.tensor(w).to('cuda'),torch.tensor(l).to('cuda')))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([2.8766], device='cuda:0', grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model((torch.tensor(w).to('cuda'),torch.tensor(l).to('cuda')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4762b0c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}