overview.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 概要\n",
    "\n",
    "pix2pixとPANはLoss以外全て共通と言えます．よって，「本家のpix2pix実装にPANのlossを付け加える」という形で実装を行いました．このnotebookでは，ネットワーク構造，学習パートの要点を説明します．これだけだと動きません．動かしたい場合はREADMEを参考にして下さい．\n",
    "\n",
    "本家のpix2pix実装: https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix  \n",
    "（厳密にはlua実装 https://github.com/phillipi/pix2pix が本家ですが，↑も同じチームの人が作ったpytorch版ということなので，こちらを「本家」と呼びます．）\n",
    "\n",
    "## 共通部分(./models/network.py)\n",
    "\n",
    "ネットワークの構造は共通です．pix2pixとPANのパラメータ数は，GもDも全く同じになります．ネットワークの構造は，torchのシーケンスをprint関数で出力したもの(./G_printed.txtと./D_printed.txt)が分かりやすいかもしれません．これらはprint出力をただ貼り付けただけのテキストファイルなので実装とは何の関係もありません．実際のGとDの実装は以下のようになります．\n",
    "\n",
    "### G\n",
    "まずGはU-Netです．Skip-connectionの表現で再帰っぽい記述を使っていますが，特に大きな意味はないと思います．（ハードコーディングを避けただけだと思います．）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import init\n",
    "import functools\n",
    "from torch.autograd import Variable\n",
    "from torch.optim import lr_scheduler\n",
    "import numpy as np\n",
    "\n",
    "def define_G(input_nc, output_nc, ngf, which_model_netG, norm='batch', use_dropout=False, init_type='normal', gpu_ids=[]):\n",
    "    netG = None\n",
    "    use_gpu = len(gpu_ids) > 0\n",
    "    norm_layer = get_norm_layer(norm_type=norm)\n",
    "\n",
    "    if use_gpu:\n",
    "        assert(torch.cuda.is_available())\n",
    "    # 今回は全部の実験でunet_256を使用\n",
    "    if which_model_netG == 'unet_128':\n",
    "        netG = UnetGenerator(input_nc, output_nc, 7, ngf, norm_layer=norm_layer, use_dropout=use_dropout, gpu_ids=gpu_ids)\n",
    "    elif which_model_netG == 'unet_256':\n",
    "        netG = UnetGenerator(input_nc, output_nc, 8, ngf, norm_layer=norm_layer, use_dropout=use_dropout, gpu_ids=gpu_ids)\n",
    "    else:\n",
    "        raise NotImplementedError('Generator model name [%s] is not recognized' % which_model_netG)\n",
    "    if len(gpu_ids) > 0:\n",
    "        netG.cuda(device_id=gpu_ids[0])\n",
    "    init_weights(netG, init_type=init_type)\n",
    "    return netG\n",
    "\n",
    "class UnetGenerator(nn.Module):\n",
    "    def __init__(self, input_nc, output_nc, num_downs, ngf=64,\n",
    "                 norm_layer=nn.BatchNorm2d, use_dropout=False, gpu_ids=[]):\n",
    "        super(UnetGenerator, self).__init__()\n",
    "        self.gpu_ids = gpu_ids\n",
    "\n",
    "        # construct unet structure\n",
    "        # 一番真ん中のレイヤ(ボトルネック部分)\n",
    "        unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True)\n",
    "        # 真ん中のレイヤをサンドイッチするようにdown，upレイヤをくっつける\n",
    "        for i in range(num_downs - 5):\n",
    "            unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer, use_dropout=use_dropout)\n",
    "        unet_block = UnetSkipConnectionBlock(ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer)\n",
    "        unet_block = UnetSkipConnectionBlock(ngf * 2, ngf * 4, input_nc=None, submodule=unet_block, norm_layer=norm_layer)\n",
    "        unet_block = UnetSkipConnectionBlock(ngf, ngf * 2, input_nc=None, submodule=unet_block, norm_layer=norm_layer)\n",
    "        # 一番外(一番左と一番右)のレイヤ．つまりinputとoutput\n",
    "        unet_block = UnetSkipConnectionBlock(output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer)\n",
    "\n",
    "        self.model = unet_block\n",
    "\n",
    "    def forward(self, input):\n",
    "        if self.gpu_ids and isinstance(input.data, torch.cuda.FloatTensor):\n",
    "            return nn.parallel.data_parallel(self.model, input, self.gpu_ids)\n",
    "        else:\n",
    "            return self.model(input)\n",
    "        \n",
    "# 再帰的に下のような構造をつくる(イメージ)\n",
    "# これまでに構成した[submodule]というのが白い四角□\n",
    "# これから構成するdownやupが黒い四角■\n",
    "# ■                ■\n",
    "# ■                ■\n",
    "# ■ □      □ ■\n",
    "# ■ □ □ □ ■\n",
    "class UnetSkipConnectionBlock(nn.Module):\n",
    "    def __init__(self, outer_nc, inner_nc, input_nc=None,\n",
    "                 submodule=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm2d, use_dropout=False):\n",
    "        super(UnetSkipConnectionBlock, self).__init__()\n",
    "        self.outermost = outermost\n",
    "        if type(norm_layer) == functools.partial:\n",
    "            use_bias = norm_layer.func == nn.InstanceNorm2d\n",
    "        else:\n",
    "            use_bias = norm_layer == nn.InstanceNorm2d\n",
    "        if input_nc is None:\n",
    "            input_nc = outer_nc\n",
    "        downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4,\n",
    "                             stride=2, padding=1, bias=use_bias)\n",
    "        downrelu = nn.LeakyReLU(0.2, True)\n",
    "        downnorm = norm_layer(inner_nc)\n",
    "        uprelu = nn.ReLU(True)\n",
    "        upnorm = norm_layer(outer_nc)\n",
    "\n",
    "        if outermost: # 一番外側(input, output)\n",
    "            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,\n",
    "                                        kernel_size=4, stride=2,\n",
    "                                        padding=1)\n",
    "            down = [downconv]\n",
    "            up = [uprelu, upconv, nn.Tanh()]\n",
    "            model = down + [submodule] + up # サンドイッチ\n",
    "        elif innermost: # 一番真ん中\n",
    "            upconv = nn.ConvTranspose2d(inner_nc, outer_nc,\n",
    "                                        kernel_size=4, stride=2,\n",
    "                                        padding=1, bias=use_bias)\n",
    "            down = [downrelu, downconv]\n",
    "            up = [uprelu, upconv, upnorm]\n",
    "            model = down + up # サンドイッチ\n",
    "        else: # 真ん中 を挟むようにしてつくる途中のレイヤ\n",
    "            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,\n",
    "                                        kernel_size=4, stride=2,\n",
    "                                        padding=1, bias=use_bias)\n",
    "            down = [downrelu, downconv, downnorm]\n",
    "            up = [uprelu, upconv, upnorm]\n",
    "\n",
    "            if use_dropout:\n",
    "                model = down + [submodule] + up + [nn.Dropout(0.5)] # サンドイッチ\n",
    "            else:\n",
    "                model = down + [submodule] + up\n",
    "\n",
    "        self.model = nn.Sequential(*model)\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.outermost:\n",
    "            return self.model(x)\n",
    "        else:\n",
    "            return torch.cat([x, self.model(x)], 1) # これがSkip-cnnectionの再帰表現"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### D\n",
    "次に，Dです．convとLLeRUとBNを使ったシンプルなTrue/False判別器です．inputは「変換前画像-変換後本物画像ペア」または「変換前画像-生成された偽物画像ペア」のどちらかを想定しています．「ペア」というのは単純に2つの画像をconcatしたものを入力することで表現しています．\n",
    "\n",
    "PANで中間層の出力を使いたいので，register_forward_hookで中間層の出力をself.intermediate_outputsに格納しています．シーケンスでhookを登録することにより，「○○Layerにデータが入ってきた時に何かを格納する/関数を実行する」みたいなことが可能になります．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class NLayerDiscriminator(nn.Module):\n",
    "\n",
    "    def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d, use_sigmoid=False, gpu_ids=[]):\n",
    "        super(NLayerDiscriminator, self).__init__()\n",
    "        self.gpu_ids = gpu_ids\n",
    "        if type(norm_layer) == functools.partial:\n",
    "            use_bias = norm_layer.func == nn.InstanceNorm2d\n",
    "        else:\n",
    "            use_bias = norm_layer == nn.InstanceNorm2d\n",
    "\n",
    "        kw = 4\n",
    "        padw = 1\n",
    "        input_conv = nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw)\n",
    "        sequence = [\n",
    "            input_conv,\n",
    "            nn.LeakyReLU(0.2, True)\n",
    "        ]\n",
    "        # register_forward_hookで中間層出力\n",
    "        input_conv.register_forward_hook(self.add_intermediate_output) \n",
    "\n",
    "        nf_mult = 1\n",
    "        nf_mult_prev = 1\n",
    "        for n in range(1, n_layers):\n",
    "            nf_mult_prev = nf_mult\n",
    "            nf_mult = min(2**n, 8)\n",
    "            intermediate_conv = nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult,\n",
    "                      kernel_size=kw, stride=2, padding=padw, bias=use_bias)\n",
    "            sequence += [\n",
    "                intermediate_conv,\n",
    "                norm_layer(ndf * nf_mult),\n",
    "                nn.LeakyReLU(0.2, True)\n",
    "            ]\n",
    "            # register_forward_hookで中間層出力\n",
    "            intermediate_conv.register_forward_hook(self.add_intermediate_output)\n",
    "\n",
    "        nf_mult_prev = nf_mult\n",
    "        nf_mult = min(2**n_layers, 8)\n",
    "        intermediate_conv2 = nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias)\n",
    "        sequence += [\n",
    "            intermediate_conv2,\n",
    "            norm_layer(ndf * nf_mult),\n",
    "            nn.LeakyReLU(0.2, True)\n",
    "        ]\n",
    "        # register_forward_hookで中間層出力\n",
    "        intermediate_conv2.register_forward_hook(self.add_intermediate_output)\n",
    "\n",
    "        last_conv = nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)\n",
    "        sequence += [last_conv]\n",
    "        # register_forward_hookで中間層出力\n",
    "        last_conv.register_forward_hook(self.add_intermediate_output)\n",
    "\n",
    "        if use_sigmoid:\n",
    "            sequence += [nn.Sigmoid()]\n",
    "\n",
    "        self.model = nn.Sequential(*sequence)\n",
    "        self.intermediate_outputs = []\n",
    "\n",
    "    def forward(self, input):\n",
    "        self.intermediate_outputs = []\n",
    "        if len(self.gpu_ids) and isinstance(input.data, torch.cuda.FloatTensor):\n",
    "            return nn.parallel.data_parallel(self.model, input, self.gpu_ids)\n",
    "        else:\n",
    "            return self.model(input)\n",
    "\n",
    "    def add_intermediate_output(self, conv, input, output):\n",
    "        self.intermediate_outputs.append(Variable(output.data, requires_grad=False))\n",
    "\n",
    "    def get_intermediate_outputs(self):\n",
    "        return self.intermediate_outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 共通しない部分(Loss)\n",
    "\n",
    "主に違うのはbackward_D( )とbackward_G( )の中だけです．\n",
    "\n",
    "### pix2pix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'BaseModel' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-d992d7510d37>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mPix2PixModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBaseModel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;31m# 初期化は省略\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'BaseModel' is not defined"
     ]
    }
   ],
   "source": [
    "class Pix2PixModel(BaseModel):\n",
    "    \n",
    "    # 初期化は省略\n",
    "    \n",
    "    def forward(self):\n",
    "        self.real_A = Variable(self.input_A)\n",
    "        self.fake_B = self.netG.forward(self.real_A)\n",
    "        self.real_B = Variable(self.input_B)\n",
    "\n",
    "    # no backprop gradients\n",
    "    def test(self):\n",
    "        self.real_A = Variable(self.input_A, volatile=True)\n",
    "        self.fake_B = self.netG.forward(self.real_A)\n",
    "        self.real_B = Variable(self.input_B, volatile=True)\n",
    "        \n",
    "    def backward_D(self):\n",
    "        # Fake\n",
    "        fake_AB = torch.cat((self.real_A, self.fake_B), 1)\n",
    "        #fake_AB --> torch.Size([1, 6, 256, 256])\n",
    "\n",
    "        # detach: make fake_AB volatile\n",
    "        self.pred_fake = self.netD.forward(fake_AB.detach())\n",
    "        self.loss_D_fake = self.criterionGAN(self.pred_fake, False)\n",
    "\n",
    "        # Real\n",
    "        real_AB = torch.cat((self.real_A, self.real_B), 1)\n",
    "        self.pred_real = self.netD.forward(real_AB)\n",
    "        self.loss_D_real = self.criterionGAN(self.pred_real, True)\n",
    "\n",
    "        # Combined loss\n",
    "        self.loss_D = (self.loss_D_fake + self.loss_D_real) * 0.5\n",
    "\n",
    "        self.loss_D.backward()\n",
    "\n",
    "    def backward_G(self):\n",
    "        # First, G(A) should fake the discriminator\n",
    "        fake_AB = torch.cat((self.real_A, self.fake_B), 1)\n",
    "        pred_fake = self.netD.forward(fake_AB)\n",
    "        self.loss_G_GAN = self.criterionGAN(pred_fake, True)\n",
    "\n",
    "        # Second, G(A) = B\n",
    "        self.loss_G_L1 = self.criterionL1(self.fake_B, self.real_B) * self.opt.lambda_A\n",
    "\n",
    "        self.loss_G = self.loss_G_GAN + self.loss_G_L1\n",
    "\n",
    "        self.loss_G.backward()\n",
    "\n",
    "    def optimize_parameters(self):\n",
    "        self.forward()\n",
    "\n",
    "        self.optimizer_D.zero_grad()\n",
    "        self.backward_D()\n",
    "        self.optimizer_D.step()\n",
    "\n",
    "        self.optimizer_G.zero_grad()\n",
    "        self.backward_G()\n",
    "        self.optimizer_G.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'BaseModel' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-d81e300a4185>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mPanModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBaseModel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreal_A\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_A\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfake_B\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnetG\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreal_A\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreal_B\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_B\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'BaseModel' is not defined"
     ]
    }
   ],
   "source": [
    "class PanModel(BaseModel):\n",
    "    def forward(self):\n",
    "        self.real_A = Variable(self.input_A)\n",
    "        self.fake_B = self.netG.forward(self.real_A)\n",
    "        self.real_B = Variable(self.input_B)\n",
    "\n",
    "    # no backprop gradients\n",
    "    def test(self):\n",
    "        self.real_A = Variable(self.input_A, volatile=True)\n",
    "        self.fake_B = self.netG.forward(self.real_A)\n",
    "        self.real_B = Variable(self.input_B, volatile=True)\n",
    "\n",
    "    def backward_D(self):\n",
    "        # Fake\n",
    "        fake_AB = torch.cat((self.real_A, self.fake_B), 1)\n",
    "        #fake_AB --> torch.Size([1, 6, 256, 256])\n",
    "\n",
    "        # detach: make fake_AB volatile\n",
    "        self.pred_fake = self.netD.forward(fake_AB.detach())\n",
    "        self.loss_D_fake = self.criterionGAN(self.pred_fake, False)\n",
    "\n",
    "        # outputs of intermediate layers\n",
    "        fake_inters = self.netD.get_intermediate_outputs()\n",
    "\n",
    "        # Real\n",
    "        real_AB = torch.cat((self.real_A, self.real_B), 1)\n",
    "        self.pred_real = self.netD.forward(real_AB)\n",
    "        self.loss_D_real = self.criterionGAN(self.pred_real, True)\n",
    "\n",
    "        # outputs of intermediate layers\n",
    "        real_inters = self.netD.get_intermediate_outputs()\n",
    "\n",
    "        # calc Parceptual Adversarial Loss\n",
    "        self.loss_PAN = 0\n",
    "        for (fake_i, real_i, lam) in zip(fake_inters, real_inters, self.pan_lambdas):\n",
    "            self.loss_PAN += self.criterionPAN(fake_i, real_i) * lam\n",
    "\n",
    "        if self.loss_PAN.data[0] > self.pan_mergin_m:\n",
    "            loss_PAN = Variable(self.Tensor(np.array([0], dtype=np.float)), requires_grad=False)\n",
    "        else:\n",
    "            loss_PAN = Variable(self.Tensor(np.array([self.pan_mergin_m], dtype=np.float)), requires_grad=False) - self.loss_PAN\n",
    "\n",
    "        # Combined loss\n",
    "        self.loss_D = (self.loss_D_fake + self.loss_D_real) * 0.5 + loss_PAN\n",
    "\n",
    "        self.loss_D.backward()\n",
    "\n",
    "    def backward_G(self, retain):\n",
    "        # First, G(A) should fake the discriminator\n",
    "        fake_AB = torch.cat((self.real_A, self.fake_B), 1)\n",
    "        pred_fake = self.netD.forward(fake_AB)\n",
    "        self.loss_G_GAN = self.criterionGAN(pred_fake, True)\n",
    "\n",
    "        # Combined loss\n",
    "        self.loss_G = self.loss_G_GAN + self.loss_PAN\n",
    "\n",
    "        self.loss_G.backward(retain_graph=retain)\n",
    "\n",
    "    def optimize_parameters(self):\n",
    "        self.forward()\n",
    "\n",
    "        # update D\n",
    "        self.optimizer_D.zero_grad()\n",
    "        self.backward_D()\n",
    "        self.optimizer_D.step()\n",
    "\n",
    "        self.optimizer_G.zero_grad()\n",
    "        self.backward_G()\n",
    "        self.optimizer_G.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}