02d_train_on_melspectrograms_pytorch_maxpool.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from birdcall.data import *\n",
    "from birdcall.metrics import *\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = pd.read_pickle('data/classes.pkl')\n",
    "train_ds = MelspecPoolDataset(pd.read_pickle('data/train_set.pkl'), classes, len_mult=60)\n",
    "valid_ds = MelspecPoolDataset(pd.read_pickle('data/val_set.pkl'), classes, len_mult=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15840, 2640)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_ds), len(valid_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = torch.utils.data.DataLoader(train_ds, batch_size=16, shuffle=True, num_workers=NUM_WORKERS)\n",
    "valid_dl = torch.utils.data.DataLoader(valid_ds, batch_size=2*16, shuffle=False, num_workers=NUM_WORKERS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([16, 10, 3, 80, 212]),\n",
       " tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [1., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         ...,\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.]], dtype=torch.float64))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for b in train_dl: break\n",
    "b[0].shape, b[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(-0.0010), tensor(1.5770))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[0].mean(), b[0].std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.cnn = nn.Sequential(*list(torchvision.models.resnet34(True).children())[:-2])\n",
    "        self.classifier = nn.Sequential(*[\n",
    "            nn.Linear(512, 512), nn.ReLU(), nn.Dropout(p=0.5), nn.BatchNorm1d(512),\n",
    "            nn.Linear(512, 512), nn.ReLU(), nn.Dropout(p=0.5), nn.BatchNorm1d(512),\n",
    "            nn.Linear(512, len(classes))\n",
    "        ])\n",
    "    \n",
    "    def forward(self, x):\n",
    "        bs, im_num, ch, y_dim, x_dim = x.shape\n",
    "        x = self.cnn(x.view(-1, ch, y_dim, x_dim))\n",
    "        x = x.mean((2,3))\n",
    "        x = self.classifier(x)\n",
    "        x = x.view(bs, im_num, -1)\n",
    "        x = x.max(1)[0]\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model().cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "import time\n",
    "\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "optimizer = optim.Adam(model.parameters(), 1e-3)\n",
    "scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 275.6] loss: 0.187, acc: 0.000, f1: 0.000\n",
      "[2, 270.2] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[4, 269.5] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[5, 269.1] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[6, 269.3] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[7, 270.0] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[8, 270.1] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[9, 269.5] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[10, 268.8] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[11, 269.2] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[12, 268.9] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[13, 269.4] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[14, 268.9] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[15, 268.2] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[16, 269.7] loss: 0.025, acc: 0.000, f1: 0.000\n",
      "[17, 269.8] loss: 0.024, acc: 0.000, f1: 0.000\n",
      "[18, 270.0] loss: 0.024, acc: 0.000, f1: 0.000\n",
      "[19, 269.0] loss: 0.024, acc: 0.000, f1: 0.000\n",
      "[20, 268.4] loss: 0.024, acc: 0.000, f1: 0.000\n",
      "[21, 269.2] loss: 0.023, acc: 0.000, f1: 0.000\n",
      "[22, 269.5] loss: 0.023, acc: 0.000, f1: 0.000\n",
      "[23, 269.5] loss: 0.023, acc: 0.000, f1: 0.000\n",
      "[24, 269.1] loss: 0.023, acc: 0.000, f1: 0.000\n",
      "[25, 269.1] loss: 0.023, acc: 0.000, f1: 0.000\n",
      "[26, 270.0] loss: 0.023, acc: 0.000, f1: 0.000\n",
      "[27, 269.1] loss: 0.022, acc: 0.000, f1: 0.000\n",
      "[28, 269.4] loss: 0.022, acc: 0.000, f1: 0.000\n",
      "[29, 269.0] loss: 0.022, acc: 0.000, f1: 0.000\n",
      "[30, 268.7] loss: 0.022, acc: 0.000, f1: 0.000\n",
      "[31, 269.8] loss: 0.022, acc: 0.000, f1: 0.000\n",
      "[32, 269.9] loss: 0.022, acc: 0.000, f1: 0.000\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-ceb7a87c3b4f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m         \u001b[0mscheduler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/optim/lr_scheduler.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     64\u001b[0m                 \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step_count\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m                 \u001b[0mwrapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m             \u001b[0;31m# Note that the returned function here is no longer a bound method,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     94\u001b[0m                 \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                 \u001b[0mexp_avg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m                 \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m                     \u001b[0;31m# Maintains the maximum of all 2nd moment running avg. till now\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(130):\n",
    "    t0 = time.time()\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(train_dl, 0):\n",
    "        model.train()\n",
    "        inputs, labels = data[0].cuda(), data[1].cuda()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "        \n",
    "        running_loss += loss.item()\n",
    "\n",
    "        if i % len(train_dl) == len(train_dl)-1:\n",
    "            model.eval();\n",
    "            preds = []\n",
    "            targs = []\n",
    "\n",
    "            with torch.no_grad():\n",
    "                for data in valid_dl:\n",
    "                    inputs, labels = data[0].cuda(), data[1].cuda()\n",
    "                    outputs = model(inputs)\n",
    "                    preds.append(outputs.cpu().detach())\n",
    "                    targs.append(labels.cpu().detach())\n",
    "\n",
    "                preds = torch.cat(preds)\n",
    "                targs = torch.cat(targs)\n",
    "            \n",
    "            accuracy = accuracy_score(preds.sigmoid() > 0.5, targs)\n",
    "            f1 = f1_score(preds.sigmoid() > 0.5, targs, average='micro')\n",
    "            print(f'[{epoch + 1}, {time.time() - t0:.1f}] loss: {running_loss / (len(train_dl)-1):.3f}, acc: {accuracy:.3f}, f1: {f1:.3f}')\n",
    "            running_loss = 0.0\n",
    "            \n",
    "            if (epoch % 10 == 0) and (epoch != 0): torch.save(model.state_dict(), f'models/{epoch}_maxpool_{round(f1, 2)}.pth')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}