LeNet_Mixup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy  as np\n",
    "from torch import optim\n",
    "import torch.nn as nn\n",
    "import pandas as pd\n",
    "from torch.utils.data import TensorDataset,DataLoader,Dataset\n",
    "from torch.autograd import Variable\n",
    "from scipy.stats import beta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build CustomedDataset\n",
    "# Separate the given dataset into two parts, four for train and one for test\n",
    "\n",
    "class CustomedDataSet(Dataset):\n",
    "    def __init__(self, train=True):\n",
    "        self.train = train\n",
    "        if self.train :\n",
    "            trainX = pd.read_csv('./train.csv',nrows=33600)\n",
    "            trainY = trainX.label.as_matrix().tolist()\n",
    "            trainX = trainX.drop('label',axis=1).as_matrix().reshape(trainX.shape[0], 1, 28, 28)\n",
    "            self.datalist = trainX\n",
    "            self.labellist = trainY\n",
    "        else:\n",
    "            testX = pd.read_csv('./train.csv')\n",
    "            testX = testX.drop(testX.head(33600).index)\n",
    "            testY = testX.label.as_matrix().tolist()\n",
    "            testX = testX.drop('label',axis=1).as_matrix().reshape(testX.shape[0], 1, 28, 28)\n",
    "            self.datalist = testX\n",
    "            self.labellist = testY\n",
    "            \n",
    "    def __getitem__(self, index):\n",
    "        if self.train:\n",
    "            return torch.Tensor(self.datalist[index].astype(float)),self.labellist[index]\n",
    "        else:\n",
    "            return torch.Tensor(self.datalist[index].astype(float)),self.labellist[index]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.datalist.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper parameters\n",
    "batch_size = 100\n",
    "learning_rate = 1e-2\n",
    "epoches = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/peng/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:6: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  \n",
      "/home/peng/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:7: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  import sys\n",
      "/home/peng/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:13: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  del sys.path[0]\n",
      "/home/peng/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:14: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "train_dataset = CustomedDataSet()\n",
    "test_dataset  = CustomedDataSet(train=False)\n",
    "train_loader  = DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True,num_workers=2)\n",
    "test_loader   = DataLoader(dataset= test_dataset,batch_size=batch_size,shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LeNet network\n",
    "class LeNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(LeNet, self).__init__()\n",
    "        self.conv=nn.Sequential(\n",
    "            nn.Conv2d(1,6,5,stride=1,padding=2),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2,2),\n",
    "            nn.Conv2d(6,16,5,stride=1,padding=2),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2,2),\n",
    "            nn.Conv2d(16,120,5,stride=1,padding=2),\n",
    "            nn.ReLU(),\n",
    "        )\n",
    "        self.fc=nn.Sequential(\n",
    "            nn.Linear(7*7*120,120),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(120, 84),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(84,10),\n",
    "            nn.Sigmoid(),\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        out=self.conv(x)\n",
    "        out=out.view(out.size(0),-1)\n",
    "        out=self.fc(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lenet = LeNet()  # network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss() \n",
    "optimizer = optim.SGD(lenet.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mixup input data\n",
    "def mixup_data(x, y,alpha):\n",
    "    lam = np.random.beta(alpha, alpha)\n",
    "    batch_size = x.size()[0]\n",
    "    index = torch.randperm(batch_size)\n",
    "    mixed_x = lam * x + (1 - lam) * x[index,:]\n",
    "    y_a, y_b = y, y[index]\n",
    "    return mixed_x, y_a, y_b, lam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The loss function will change of course\n",
    "def mixup_criterion(y_a, y_b, lam):\n",
    "    return lambda criterion, pred: lam * criterion(pred, y_a) + (1 - lam) * criterion(pred, y_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alpha 0.1, Epoch [1/5], Loss: 1.4764\n",
      "Alpha 0.1, Epoch [2/5], Loss: 1.4724\n",
      "Alpha 0.1, Epoch [3/5], Loss: 1.4893\n",
      "Alpha 0.1, Epoch [4/5], Loss: 1.4717\n",
      "Alpha 0.1, Epoch [5/5], Loss: 1.5857\n",
      "Accu rate at Alpha=0.1: 98.524\n",
      "Alpha 0.2, Epoch [1/5], Loss: 1.4714\n",
      "Alpha 0.2, Epoch [2/5], Loss: 1.4893\n",
      "Alpha 0.2, Epoch [3/5], Loss: 1.6636\n",
      "Alpha 0.2, Epoch [4/5], Loss: 1.5162\n",
      "Alpha 0.2, Epoch [5/5], Loss: 1.4688\n",
      "Accu rate at Alpha=0.2: 98.262\n",
      "Alpha 0.4, Epoch [1/5], Loss: 1.7423\n",
      "Alpha 0.4, Epoch [2/5], Loss: 1.6485\n",
      "Alpha 0.4, Epoch [3/5], Loss: 1.7026\n",
      "Alpha 0.4, Epoch [4/5], Loss: 1.5097\n",
      "Alpha 0.4, Epoch [5/5], Loss: 1.4793\n",
      "Accu rate at Alpha=0.4: 98.464\n",
      "Alpha 0.8, Epoch [1/5], Loss: 1.4854\n",
      "Alpha 0.8, Epoch [2/5], Loss: 1.7218\n",
      "Alpha 0.8, Epoch [3/5], Loss: 1.6794\n",
      "Alpha 0.8, Epoch [4/5], Loss: 1.6756\n",
      "Alpha 0.8, Epoch [5/5], Loss: 1.6503\n",
      "Accu rate at Alpha=0.8: 98.476\n"
     ]
    }
   ],
   "source": [
    "for alpha in [0.1,0.2,0.4,0.8]:\n",
    "    for epoch in range(epoches):\n",
    "        for i, (x,y) in enumerate(train_loader):\n",
    "            img, y_a, y_b, lam = mixup_data(x, y, alpha)\n",
    "            img = Variable(img)\n",
    "            y_a = Variable(y_a)\n",
    "            y_b = Variable(y_b)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            output=lenet(img)\n",
    "            loss_func = mixup_criterion(y_a, y_b, lam)\n",
    "            loss = loss_func(criterion, output)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            if (i+1) % 336 == 0:\n",
    "                print ('Alpha %.1f, Epoch [%d/%d], Loss: %.4f' %(alpha, epoch+1, epoches, loss.item()))\n",
    "    \n",
    "    ans = torch.LongTensor()\n",
    "    for (images,labels) in test_loader:\n",
    "        images = Variable(images)\n",
    "        outputs = lenet(images)\n",
    "        _,predicted = torch.max(outputs.data, 1)\n",
    "        ans = torch.cat((ans,predicted),0)\n",
    "    \n",
    "    ans = ans.numpy()\n",
    "    \n",
    "    corr=0\n",
    "    for i,(images,labels) in enumerate(test_dataset):\n",
    "        if ans[i]==labels:\n",
    "            corr=corr+1\n",
    "    print(\"Accu rate at Alpha=%.1f: %.3f\"%(alpha,corr/84))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}