SimCLR_ImageNet_Subset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "SimCLR_ImageNet_Subset.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sayakpaul/SimCLR-in-TensorFlow-2/blob/master/SimCLR_ImageNet_Subset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LMp5lKNjgBXq",
        "colab_type": "text"
      },
      "source": [
        "## Initial setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lIYdn1woOS1n",
        "outputId": "4d8be820-c86b-4f28-e2b8-724812dafd5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0-rc4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "dZowsDvOYK37",
        "outputId": "c77a9da7-b3ec-4473-b047-1e242baf2433",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sat May  9 03:35:44 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.82       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   36C    P0    25W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I2MOoF9zgMHp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Install wandb for experiment tracking\n",
        "!pip install --upgrade https://github.com/wandb/client/archive/feature/code-save.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EXx0RnrigMwd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import wandb\n",
        "wandb.login()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "421MrJNMYQD7",
        "colab": {}
      },
      "source": [
        "# Other imports\n",
        "from tensorflow.keras.layers import *\n",
        "from tensorflow.keras.models import *\n",
        "import matplotlib.pyplot as plt\n",
        "from imutils import paths\n",
        "from tqdm import tqdm\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import cv2\n",
        "\n",
        "# Random seed fixation\n",
        "tf.random.set_seed(666)\n",
        "np.random.seed(666)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UUFlGxuJgBX9",
        "colab_type": "text"
      },
      "source": [
        "## Dataset gathering and preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "cq8xwssDYNTq",
        "colab": {}
      },
      "source": [
        "# Gather dataset\n",
        "!git clone https://github.com/thunderInfy/imagenet-5-categories"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Z0atoQ0MYT4b",
        "outputId": "11041267-2c59-420c-8a8b-ad519aecc880",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# How many training images for SimCLR?\n",
        "train_images = list(paths.list_images(\"imagenet-5-categories/train\"))\n",
        "print(len(train_images))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1250\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Ad9dX7ONYV9Y",
        "colab": {}
      },
      "source": [
        "# Augmentation utilities (differs from the original implementation)\n",
        "# Referred from: https://arxiv.org/pdf/2002.05709.pdf (Appendxi A \n",
        "# corresponding GitHub: https://github.com/google-research/simclr/)\n",
        "\n",
        "class CustomAugment(object):\n",
        "    def __call__(self, sample):        \n",
        "        # Random flips\n",
        "        sample = self._random_apply(tf.image.flip_left_right, sample, p=0.5)\n",
        "        \n",
        "        # Randomly apply transformation (color distortions) with probability p.\n",
        "        sample = self._random_apply(self._color_jitter, sample, p=0.8)\n",
        "        sample = self._random_apply(self._color_drop, sample, p=0.2)\n",
        "\n",
        "        return sample\n",
        "\n",
        "    def _color_jitter(self, x, s=1):\n",
        "        # one can also shuffle the order of following augmentations\n",
        "        # each time they are applied.\n",
        "        x = tf.image.random_brightness(x, max_delta=0.8*s)\n",
        "        x = tf.image.random_contrast(x, lower=1-0.8*s, upper=1+0.8*s)\n",
        "        x = tf.image.random_saturation(x, lower=1-0.8*s, upper=1+0.8*s)\n",
        "        x = tf.image.random_hue(x, max_delta=0.2*s)\n",
        "        x = tf.clip_by_value(x, 0, 1)\n",
        "        return x\n",
        "    \n",
        "    def _color_drop(self, x):\n",
        "        image = tf.image.rgb_to_grayscale(x)\n",
        "        image = tf.tile(x, [1, 1, 1, 3])\n",
        "        return x\n",
        "    \n",
        "    def _random_apply(self, func, x, p):\n",
        "        return tf.cond(\n",
        "          tf.less(tf.random.uniform([], minval=0, maxval=1, dtype=tf.float32),\n",
        "                  tf.cast(p, tf.float32)),\n",
        "          lambda: func(x),\n",
        "          lambda: x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "XKYH6UxnYa7g",
        "colab": {}
      },
      "source": [
        "# Build the augmentation pipeline\n",
        "data_augmentation = Sequential([Lambda(CustomAugment())])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4eDTVhuHYdwi",
        "colab": {}
      },
      "source": [
        "# Image preprocessing utils\n",
        "@tf.function\n",
        "def parse_images(image_path):\n",
        "    image_string = tf.io.read_file(image_path)\n",
        "    image = tf.image.decode_jpeg(image_string, channels=3)\n",
        "    image = tf.image.convert_image_dtype(image, tf.float32)\n",
        "    image = tf.image.resize(image, size=[224, 224])\n",
        "\n",
        "    return image"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "otqf-I5FZLqd",
        "colab": {}
      },
      "source": [
        "# Create TensorFlow dataset\n",
        "BATCH_SIZE = 64\n",
        "\n",
        "train_ds = tf.data.Dataset.from_tensor_slices(train_images)\n",
        "train_ds = (\n",
        "    train_ds\n",
        "    .map(parse_images, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
        "    .shuffle(1024)\n",
        "    .batch(BATCH_SIZE, drop_remainder=True)\n",
        "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2avjHJzRgBYS",
        "colab_type": "text"
      },
      "source": [
        "## Utilities"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "F1_Wl0DzZQ5F",
        "colab": {}
      },
      "source": [
        "# Architecture utils\n",
        "def get_resnet_simclr(hidden_1, hidden_2, hidden_3):\n",
        "    base_model = tf.keras.applications.ResNet50(include_top=False, weights=None, input_shape=(224, 224, 3))\n",
        "    base_model.trainabe = True\n",
        "    inputs = Input((224, 224, 3))\n",
        "    h = base_model(inputs, training=True)\n",
        "    h = GlobalAveragePooling2D()(h)\n",
        "\n",
        "    projection_1 = Dense(hidden_1)(h)\n",
        "    projection_1 = Activation(\"relu\")(projection_1)\n",
        "    projection_2 = Dense(hidden_2)(projection_1)\n",
        "    projection_2 = Activation(\"relu\")(projection_2)\n",
        "    projection_3 = Dense(hidden_3)(projection_2)\n",
        "\n",
        "    resnet_simclr = Model(inputs, projection_3)\n",
        "\n",
        "    return resnet_simclr"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7-hwmH_LZc7K",
        "colab": {}
      },
      "source": [
        "!wget https://raw.githubusercontent.com/sthalles/SimCLR-tensorflow/master/utils/helpers.py\n",
        "!wget https://raw.githubusercontent.com/sthalles/SimCLR-tensorflow/master/utils/losses.py"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "X6d1jsrs7ztp"
      },
      "source": [
        "Don't forget to comment the `augmentation` import in the `helpers.py` script."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "2OUhSNfTZgY6",
        "colab": {}
      },
      "source": [
        "from losses import _dot_simililarity_dim1 as sim_func_dim1, _dot_simililarity_dim2 as sim_func_dim2\n",
        "import helpers"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Y0Ae5dMLZney",
        "colab": {}
      },
      "source": [
        "# Mask to remove positive examples from the batch of negative samples\n",
        "negative_mask = helpers.get_negative_mask(BATCH_SIZE)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "K6qPdhPrZpFA",
        "colab": {}
      },
      "source": [
        "@tf.function\n",
        "def train_step(xis, xjs, model, optimizer, criterion, temperature):\n",
        "    with tf.GradientTape() as tape:\n",
        "        zis = model(xis)\n",
        "        zjs = model(xjs)\n",
        "\n",
        "        # normalize projection feature vectors\n",
        "        zis = tf.math.l2_normalize(zis, axis=1)\n",
        "        zjs = tf.math.l2_normalize(zjs, axis=1)\n",
        "\n",
        "        l_pos = sim_func_dim1(zis, zjs)\n",
        "        l_pos = tf.reshape(l_pos, (BATCH_SIZE, 1))\n",
        "        l_pos /= temperature\n",
        "\n",
        "        negatives = tf.concat([zjs, zis], axis=0)\n",
        "\n",
        "        loss = 0\n",
        "\n",
        "        for positives in [zis, zjs]:\n",
        "            l_neg = sim_func_dim2(positives, negatives)\n",
        "\n",
        "            labels = tf.zeros(BATCH_SIZE, dtype=tf.int32)\n",
        "\n",
        "            l_neg = tf.boolean_mask(l_neg, negative_mask)\n",
        "            l_neg = tf.reshape(l_neg, (BATCH_SIZE, -1))\n",
        "            l_neg /= temperature\n",
        "\n",
        "            logits = tf.concat([l_pos, l_neg], axis=1) \n",
        "            loss += criterion(y_pred=logits, y_true=labels)\n",
        "\n",
        "        loss = loss / (2 * BATCH_SIZE)\n",
        "\n",
        "    gradients = tape.gradient(loss, model.trainable_variables)\n",
        "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
        "\n",
        "    return loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nSs5WYn7g5v_",
        "colab_type": "code",
        "outputId": "dbce6c0f-13df-435e-d645-12a07041f9ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "wandb.init(project=\"simclr\", id=\"simclr-learning\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/sayakpaul/simclr\" target=\"_blank\">https://app.wandb.ai/sayakpaul/simclr</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/sayakpaul/simclr/runs/simclr-learning\" target=\"_blank\">https://app.wandb.ai/sayakpaul/simclr/runs/simclr-learning</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "W&B Run: https://app.wandb.ai/sayakpaul/simclr/runs/simclr-learning"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zfp8gT4ZZtif",
        "colab": {}
      },
      "source": [
        "def train_simclr(model, dataset, optimizer, criterion,\n",
        "                 temperature=0.1, epochs=100):\n",
        "    step_wise_loss = []\n",
        "    epoch_wise_loss = []\n",
        "\n",
        "    for epoch in tqdm(range(epochs)):\n",
        "        for image_batch in dataset:\n",
        "            a = data_augmentation(image_batch)\n",
        "            b = data_augmentation(image_batch)\n",
        "\n",
        "            loss = train_step(a, b, model, optimizer, criterion, temperature)\n",
        "            step_wise_loss.append(loss)\n",
        "\n",
        "        epoch_wise_loss.append(np.mean(step_wise_loss))\n",
        "        wandb.log({\"nt_xentloss\": np.mean(step_wise_loss)})\n",
        "        \n",
        "        if epoch % 10 == 0:\n",
        "            print(\"epoch: {} loss: {:.3f}\".format(epoch + 1, np.mean(step_wise_loss)))\n",
        "\n",
        "    return epoch_wise_loss, model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "atu93xMCgBYj",
        "colab_type": "text"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "QKAoeXwJZ5S1",
        "outputId": "c5f29cbc-00ff-42f1-dc95-d4922de49238",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 645
        }
      },
      "source": [
        "criterion = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, \n",
        "                                                          reduction=tf.keras.losses.Reduction.SUM)\n",
        "decay_steps = 1000\n",
        "lr_decayed_fn = tf.keras.experimental.CosineDecay(\n",
        "    initial_learning_rate=0.1, decay_steps=decay_steps)\n",
        "optimizer = tf.keras.optimizers.SGD(lr_decayed_fn)\n",
        "\n",
        "resnet_simclr_2 = get_resnet_simclr(256, 128, 50)\n",
        "\n",
        "epoch_wise_loss, resnet_simclr  = train_simclr(resnet_simclr_2, train_ds, optimizer, criterion,\n",
        "                 temperature=0.1, epochs=200)\n",
        "\n",
        "with plt.xkcd():\n",
        "    plt.plot(epoch_wise_loss)\n",
        "    plt.title(\"tau = 0.1, h1 = 256, h2 = 128, h3 = 50\")\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 1/200 [00:34<1:54:56, 34.66s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 1 loss: 4.766\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  6%|▌         | 11/200 [02:53<45:35, 14.48s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 11 loss: 3.950\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 10%|█         | 21/200 [05:14<42:24, 14.21s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 21 loss: 3.400\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 16%|█▌        | 31/200 [07:33<39:07, 13.89s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 31 loss: 2.974\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 20%|██        | 41/200 [09:52<36:47, 13.88s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 41 loss: 2.626\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 26%|██▌       | 51/200 [12:11<34:26, 13.87s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 51 loss: 2.361\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 30%|███       | 61/200 [14:29<32:08, 13.88s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 61 loss: 2.220\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 36%|███▌      | 71/200 [16:48<29:51, 13.88s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 71 loss: 2.068\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 40%|████      | 81/200 [19:07<27:29, 13.86s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 81 loss: 1.987\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 46%|████▌     | 91/200 [21:26<25:14, 13.90s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 91 loss: 1.913\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 50%|█████     | 101/200 [23:45<22:57, 13.91s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 101 loss: 1.842\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 56%|█████▌    | 111/200 [26:04<20:35, 13.89s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 111 loss: 1.792\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 60%|██████    | 121/200 [28:23<18:14, 13.86s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 121 loss: 1.743\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 66%|██████▌   | 131/200 [30:41<15:56, 13.87s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 131 loss: 1.702\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 70%|███████   | 141/200 [33:00<13:38, 13.87s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 141 loss: 1.670\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 76%|███████▌  | 151/200 [35:18<11:19, 13.86s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 151 loss: 1.635\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 80%|████████  | 161/200 [37:37<09:00, 13.86s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 161 loss: 1.611\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 86%|████████▌ | 171/200 [39:56<06:42, 13.87s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 171 loss: 1.596\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 90%|█████████ | 181/200 [42:14<04:23, 13.88s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 181 loss: 1.586\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 96%|█████████▌| 191/200 [44:34<02:05, 13.92s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "epoch: 191 loss: 1.572\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 200/200 [46:39<00:00, 14.00s/it]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEPCAYAAABbbZ8rAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeXhU1d0H8O+dfd8zWSDsBIJsosgi4i4urNal2NbtVawV2rpUrW+VupTa12oXcSuiVFuhVClWLbiibIKgCAQIWwgESDLJ7Pt63j/ivU1IMjMJyZ1J+H2eZx5llnvOnNz53nPPXQ7HGGMghBDS40hyXQFCCCGdQwFOCCE9FAU4IYT0UBTghBDSQ1GAE0JID0UBTgghPdQZE+DJZBJVVVXwer25rgrJA8lkMtdVIOS0nREBnkgk8MMf/hCDBw/GiBEjcOzYsaw+F4lEurlm7QsGg1i6dCm2bt3a7nsqKysxdepU3HfffQiHw91an1WrVmHy5Ml46aWXurWctkSjUbz66qtYuXIlsrlsIRKJIB6Pt/ladXU1br31VvTr1w9Hjx7t6qq26+jRo3jxxRexZcsW0cp86KGHcPnll2PDhg2tXsumTaPRKHw+H23svhMKhdpti2+//RZ/+ctfxO8gsh7G6/WylStXZnwcPnyYMcZYKpVic+fOZQDYueeeywCwOXPmsFQqlbEsnU7H3njjje7+Sq288sorzGq1MgAMALv//vtZMpls9b433nhDeM+jjz7aqbK2b9/OnnzyyYzvu+OOOxgAxnEcW7t2bafK6ozDhw+zq666SvieH374ofBaY2Nji7/5H//4RzZ9+nSm0WjY4MGD2caNG1ss6+jRo6xv374MAJs6dWpW60B7FixYwI4dO5bVe9etW8dKS0sZAGY2m5nH4+l0uR3Rr18/BoD16dOHHT9+XHj+1DY99e+ZSqXY4sWLmdlsZgBYeXk527x5c6fr8dprr7HVq1dnfJ/f72f//Oc/W/xNly5dymbPns3+8Ic/sGg02uk6dERNTU2LOixatIhdffXVTCqVsnPOOYd9++23wntdLhebM2eO0JZms5lt375dlHoyxliPC/Ddu3cziUQiNFh7jyeeeIIxxtiGDRsYAPbb3/6WMcbY/PnzGQD20UcfCctMJpNsy5YtrUKyX79+bODAgSwSiYj2/f7xj38I30Gj0bD//d//ZQqFgs2ePbtV4AQCAWaxWBgAplAo2NGjRztc3sqVK9mFF17Y5mt1dXVsypQpjDHGtm7dKtRr2LBhHS6nM3bv3s1sNhsDwC699FIGgP3iF78QXr/nnnvSrgMFBQUskUgI77/rrrsYAHbnnXe2eL4z+vfvz7Zt29bma3fddRf79NNPGWOMLV26lHEcxwwGg9CBWLdu3WmVna2nnnpKaIsFCxYwxjK3qc/nY1OnTmUAmMlkYueffz7jOI7p9Xp28ODBTtXjxz/+MXv88cfbfG3lypXsl7/8JWOMsWuuuSbt37OioqJT5XfUzJkz09ZjzJgxwnv5tgLALrzwQjZjxgzGcVyLfOlOPS7AGWOssrKS/fa3v2VbtmxhDz30EAPApk2bxhYtWsQWLVrElixZwoLBIGOMseeee44BYEeOHGGMMVZbW8sAsCuvvFJYHh+ay5Yta1HOX//6VwaAvfrqq1nXzePxsIqKCuGxYcMG9pvf/IaFw+GsPj99+nQ2ceJEVlVVJWzJP/jgA6ZQKISNUnPz5s0TVqBTe5zZWLFiRbsBfuzYMaZUKoV/Dx8+nAFgEokk6+XX1NS0aI+VK1eyN998M6vPjhw5ktlsNrZ161YWjUaZwWBoETbXXnutEESLFi1ir7/+Ojt27BirqalhNTU1LXq6oVCIlZSUtFgXTkdpaWm7AX7jjTey559/nlVXVzOO49jUqVNZfX09+/TTTzsc4MeOHWvRfitWrGBvvfVWVp89cuQI4ziuxfqerk1TqZQQSEOHDmV79uxhjDF23333MQDsoosuyrrezc2bN6/dAF+2bBm76qqrGGOMffjhh8JveN26dWzVqlXCut3e59sSCoVatNn27dvZ448/zhoaGjJ+NpVKsfHjxzMA7Nprr2WLFi1i//jHP4R1qqamhgUCAcYYYwcPHmRSqZQtXryY7d+/nx09epTFYjF23XXXMbPZzA4dOpR1nTurRwZ4c2vXrmUA2O7du9t8/bLLLmNSqZT5fD7huTvuuIMplUrhD3r//fczAGzp0qWtPj9kyBBWUlKSVe9j//79LYY+mj82bdqU8fN79+5lANjzzz/f6rVLL72UTZs2rdXzH3/8sVBGZ7b6t956K7v66qvbfO3UAH/00UeFskKhUMZlL126lMlkslZtYTQas6rbihUr2AcffMAYY+zrr79mANjrr78uvK5Wq5nZbM5qD+kXv/iFEEKnM3TCWFMvFoAQcKfiAzwej7Mnn3ySVVVVMcYYe/LJJzu0p/Tyyy8zqVTaqv1sNlvWdZ0wYYIwZMRYU2fl/fffZ4z9t01fe+01xhgTNjAlJSUthiu++OILBoDJ5fKsy22uX79+7Pe//32brzUP8OYOHz7Mrr766lZ7CJn4/X5hmOzUx6kdtLY0NjYKw0ZtDVs2N3/+fGYymZjL5Wrx/L59+xgA9t5772Vd786SoZeQSFofj/X7/diyZQsuuOAC6PV6MMbw8ccfQyKRIBqNIhAIwGaz4csvv4TdbsfcuXNbLeOGG27AokWL8MQTT+CNN95IWwelUol4PI4rrrgCdrsdEyZMwNChQ6HRaDBp0qSM3+Htt9+G1WrF7bff3uq1eDyOsrKyVs9feuml0Ol0CAQCWL16NS6//PKM5TTncrkwY8aMNl+LRqNIJpNIJBKQyWSYM2cOnnzySQDAv//9b9x4441ply2TyaBUKnHddddBoVBgxowZ0Ov16N+/f1Z145cfi8Vw1113YcqUKS3+Rv369cP+/fsxZcoUFBUVYcGCBbjiiivaXNbHH38MAHjyySfBcVxW5bfH7XYLB8TbEo1GEYvFIJPJ8Ktf/QoA8M033+D555/HI488gn79+mVVjlwuh1qtxvTp06FUKjFz5kxotVoMGDAg67rOnj0bW7duxfr16+F2u3HDDTcAaNmmN910EwDAYDBAKpWioaEBL774IqxWK5LJJBYtWgQAuOeee7Iut7lM61gsFmv1/MUXXyycbHDo0CHU1NSgtLQ0Y1lSqRSMMUyaNEn4G40bNw5SqRQXX3xxxs9rNBpYrVbs27cPU6dORVFRER544AFMnDixxfsYY/jnP/+JO++8E2azucVr/AH0jvydOq3bNxHdjO+Bt9Ub4odL8N2By9GjR7fYIv/9739nDoeDAWDXXHNNm8t///33GQD2ox/9qEP1SqVSbN26dWzRokXsrrvuYl6vN+Nn7r77bjZ48OBWz69atYpxHMc+//zzNj9nMpkYAHbXXXd1qI6MNY33LVy4kDHGWCwWYx999BFbs2YN+9vf/sZGjx7NOI5jdXV1jDHGdu3a1aLtOsLr9bL//Oc/bMGCBezpp5/u0GdXrFjR5l7MY4891qqXNW/evFbDVZFIhPXp04cBYKtXr2Zr1qzJao+oPevXr2d2u13Yld63bx9bs2YN+89//sPmz5/PJBIJe/DBB1t85uabb2bFxcVZ7bm0xe12sw8++IDdc8897Jlnnsn6c88884zQNs2HENpr07ffflsYdmn+UKlULfZiO0Kn07EvvviCMcZYQ0MDW7NmDVuzZg179tlnmd1uZyNHjmz1mdmzZ7Np06axSZMmMQCsuLi4U8cOtmzZwp5//nl2yy23ZL3nc/vtt7f47nK5nD388MMtjpskk0kGgP3hD39o9flLL72UjRo1qsN17YwzJsABsMGDB7MHH3yQff/73xfG1U6cOJE2wO+8886sxy0jkQhbuHAhu/rqq5nRaGxRdnvjpc3dfffdbODAgS2eq6ysZMXFxWzevHnCcwcPHmSxWEz496kB/sorrwhDD5nwB2ysVivT6/UMACsqKmJnn302M5vNLYZQOhrghw4dYvfeey8bM2YMU6vVwmcLCwuzqhtjjJ08eZINHTqUnXvuua3OQkilUmzt2rXsueeeY8899xybMWOGsLFubt26dW3uUk+YMIF9+eWXjLGm4a+FCxdmFbDr169nAJhWq2U2m41xHMekUikbM2aMcJyg+TDYRx99xJRKpXCwLlsHDhxg9957Lxs1alSL9uvbt2/Wy2grwE+ePMnKysrabNNwOCysTxqNho0aNUr4PF//UCjEHnvsMbZ///6s6qDT6ZhMJmM2m42pVCphjH3s2LFMLpe3OYTCS6VS7P/+7/+EYx2ZpFIp9txzz7Ebb7xROMDPP5YvX55VfROJBHvnnXfYc889x5599lnhuMBPf/pT4T18gD/77LMtPrto0SImkUjYjh07sirrdJ0RAX7rrbeyvXv3CltQn8/HZDIZmzhxYtoA379/P1Or1Wz+/PlZ1aW2tpaVl5ezvn37sgEDBrCf//zn7OGHH2YrV67M6vOLFy9mANjLL7/Mjh07xpYvX85KSkrYzJkzhWCpq6tjHMexu+++W/jcqQF+1VVXMalUmlWPY+bMmWzQoEFsxYoVbM+ePWzPnj3CAeCnnnrqtAL83XffZaWlpaxv377s3HPPZQ899BB7+OGH2c6dO7NqD74OAJjD4cjq/VarlfXp06fFc/zZDZdffjnbuXMnW7hwIRs4cCADwHQ6HYtEIsL473333ZexDD7AFyxYwHbu3Mn27NkjnFIYiURaBHgqlWKTJk1i48eP7/DY+zvvvCO033nnnSe0X0fOxmgrwNO16QsvvCCE98mTJxljjG3evFkYi//kk09YOBxmBoOBDR8+PKs66HQ6dskll7B169axPXv2sH379gltcckll7QI8Mcee4zNnz+fffHFF+zYsWNsz549bMaMGUwul2c1phyLxdiECRNY3759WWlpKZs3bx57+OGH2UsvvdTpYx+hUIgBYBMnThSeS6VSbPjw4aysrIzt2rWL7d+/n919991Mp9N1eO/0dPT4MXCW5sIO/rUZM2agvLxceF6v12P48OFwuVwwGo2wWCw4cuQIUqmUMJYej8dx5513QqlU4qGHHsqqLkVFRdi7d2+nv8ttt92Gt99+Gz/+8Y8BNI0hL168GHfddZfwnkAgAIPBgFtvvbXNZUQiEXz99dcYNGgQiouLsyr3wQcfbHM8u63jCh0xc+ZMzJw5s9Of37hxI5566ikAwFtvvQWNRgMAmD59epvfjR+vPxV/cUVRURFGjx6N0aNH44EHHkBRURHC4TBSqRQ2b94MAJg8eXJWdRsyZAj+/Oc/t3r+1DZ76qmn8OWXX2LQoEFYsmQJOI6DSqXC3LlzIZOl//lde+21uPbaa7OqT7YytSnfVlKpFIWFhQCAqqoqJJNJjBo1CmPHjkVFRQV8Ph+uu+66rMt9+eWXMXTo0FbPn9peNTU1ePPNN7F48WLhObVajVWrVmH69OkZy5HL5V1+sVRbY/Qcx+GFF17ArFmzMHr0aABAWVkZ9u7dm9VYfZcRbVPRTX75y18yAG3uzq1YsYJxHMfi8Xir1x5//HFWVlbGGGPslltuYQDYqlWrGGNNW/Hrr78+6yPXXSmVSrH33nuP/fznP2fV1dVZfYY/8+Xpp59m27ZtY3PmzMm6lztz5ky2ZMmSNl9btmwZ02q1wr/37NnDgKaLeTZs2JDV8k/HwoUL2xz6+MlPfsKcTidbvXo1W716Nfv444/ZyZMn2U033dTm8Yr33nuPAU3n6SaTSbZ371527733MgDsnnvuYYw1nVP+1FNPZdVLW79+fbvnwieTSWa329lLL73EGGPCud+nPvgzQbobfxptQUEBC4VCaduUsaax9rPOOosBYJMmTWKzZ89mCoWCjRo1Suixv/baa+yWW27J+oIknU4nXFh3qptvvpnNmDGjxXOxWIy98MILbNasWezll1/Oeu+rKxw/flxYrzZu3MgOHz7Mpk2bxgCwRx55pNX7nU4ne+yxx9iSJUtO+9qCzujxAb58+XJ2yy23tPnDW7ZsGZPJZG1+zuFwCAeDfD4fM5vNTCqVMr1eL4zTdeTc01wKBoPM6/V2ahdx5syZbZ4+yVjTcAB/ChzP5/MJQyzdzeFwsBdffJG9//77zOPxsMOHD7MXX3yRHTt2jH3++ectAog/8NY8aJrXuby8nAFgarVaeO+cOXNaHEvI1vr161l5eXm7rx84cED4MX/99dfshRdeYDt27GBer5dt3LiRvfrqq6JdVZhIJJjX6xVOtUzXprz6+no2f/58NmnSJDZp0iQ2ffr00wpRnU7X7rn3NTU1zO/3d3rZXe3NN99sc7266KKLhIPW+YRjrPfOickYQ21tLUpKSjK+d+fOnfjLX/6Cjz76CEOHDsUjjzyCKVOmiFDL3Nq2bRtGjRoFlUqV66p0WHV1Nd555x1s3boVO3bswAUXXIDf/e53KCgoaPXe+vp6PPTQQ9i0aRPOPfdcPPDAAxgzZkzGYYy21NfXw+v1tnlaJ2lt06ZNmDx58mmfvikGxhgOHDiAd955B5s3b8b+/ftxzTXX4De/+Q20Wm2uq9dKrw5wQgjpzXr8QUyxMMbg9XrhdDrh9XoRDAbh9XrhdrvhdDrh9/uFixJisRji8ThCoRCCwSDC4TBisRgSiUSru5lxHAepVAqZTAaFQgG5XA6ZTAa5XA65XA6NRgOLxQKDwQC9Xg+j0QitVguTyQSj0QiVSgWVSgWtVguj0Qi5XJ6jFupeiUQCHo8HgUAAwWAQfr8fSqUSBoMBJpMJcrkcwWAQ33zzDbZt24ZgMIhQKCT8PSZMmICLL74YRqMRPp8Pn332GdasWYNAICAc7OZ7iHy7N29bpVIJuVwOo9GIkSNHQqfTwefzCX8Tg8EAu90Oo9HYI3qabfH7/XC5XAgGg8IjFArB7/fD7/cjEAi0+H/+ro+RSATRaBTxeByxWKzFOj5s2DAMGjQI3377LSKRCNRqNfR6vfAwGAxC+5lMJphMJuH/zWZzr1ifo9EoTp48CbfbDZfLhfr6evh8PgSDQUQiEYRCIYRCIUSjUSEv4vG4kBepVAqjR4/GM88802rZovXAf/azn6GiogJqtRomkwkWi0VY+dVqNXQ6Hcxms/DHtFgssFgs0Gq1ndrNbUsqlUI4HIbf74fP50MoFILP54PP50MgEEB9fT3q6+tRV1cHp9MpvOZ2u1FbW5vx9rIcx0GhUAgPtVoNrVYLtVoNpVIJqVQKqVQKjuPAcRwYY0ilUsLZE3zwJxIJxONxYSPg8XiQSqWy+o4qlQomkwlWqxU6nQ5arRYWiwU2m034YdjtdlitVmi1WuEHxP9w1Gp1lwdQLBZDQ0MDXC6X8ON3Op1wOp1CEAQCAbjdbvh8Pni9Xvj9fiFEAoEAGhsbs24DoOnMBbVaDYVCAaVSCZVKJWwc+YdEIhEevFQqhXg8jnA4jHA4LPzA+LBq64yE5hQKBex2OwoKCmC321FcXIzCwkIUFhZCo9HAZDLBZrPBbDbDZrPBZDJBp9Od9hk/PMYYotGo0HngQ5jvfNTW1qKurk74b11dHVwul/C3yIZSqYROp4NarYZMJoNKpRI2cAqFQljHgab25NftWCyGSCQi/P6yuQWyRqOBTqeDXq8X2tRqtcJisUCj0aCgoAA2m01Y141GI8xms7Ax6Ip2ZYwhFoshFAohEAjA5/OhoaEBbrdb+Df/nfhOXW1tLRoaGuBwONDQ0JB2+VKpFBqNBkqlUsiL5uuqVCrFuHHj8MILL7T6rKgBvn37dkQiEbhcLng8Hvj9/qzuNSyXy6FUKqFQKKDRaITekVKpFL6gRCIRwpD/ocXjcSEA+B9hJlKpFHa7HXa7XdjAmEwmFBUVobi4GDabTegF86cgms1mGAwGyGSybul9pVIpoSfk8XgQDAbh8Xjg9XoRiUQQiUSEPQK+F+VyuYTeqtPphMvlgs/nQzQazfj9tVqtsAHif6T8HoFEIhE2RPyPI5lMttgI8XWKxWIIBAJZBQMfbkajEUajEXq9HhqNBlqtFnq9XvibaLVa4Tl+Zecf/A9dpVJ1WSCeKh6Pw+fzwePxCD9cr9cLr9eL+vp6OBwOOBwONDY2CiHpcDjavT850LTh5zeefAjK5XJhHecDUSKRgOM4pFIppFIpxGIxhMNhIVj43lumn7REIoHdbkdJSQmKiopgs9lgsVhQUlICq9UqtLtWq4VGoxH2/nQ6HXQ6XZf1ipPJZIsNtsfjEdrV4/HA7XYLOeH3+4V2bWhogMfjQSgUSrt8vl21Wq3QrnyO8AEplUqFuvDrcDQaRTQaRTgcFvb6solJmUwm5EVhYaHQtn369EGfPn2EDXdhYSGMRqOQY3K5vNO5kdMxcMYYQqEQwuGw0APzer3w+XxobGyE2+0WehD88AS/u8HvtvG7GYwxYTii+Y+AX+n43rBGoxF23/geqMFggE6nQ0FBAaxWa4/dBc5GKBSCw+EQ2pYPn+aBFAgEhHDge578g99I8m0OQAh1fhiIH3pQKBTQ6XSwWCxCT4kPArPZjIKCAmi12m4N3HyQSqWEXWZ+N5rfA2ne/vyuM9/54Ndxvq35Bx/mSqWyxcaLX7/5dZ3/N7+eW61WYUPYG9o7lUqhsbFR2HtoPqzp8XiEjmIwGBTWX75jwe/p8nt1zddhpVIJpVIpdCp0Oh1UKpWQHXxbWiwW6HQ6YQPXHXuvvJKSEsyaNavVhCqiBnhZWRkuvPBCLFmyRKwiCSGkxxs0aBAmT56Mv/3tby2eF3UzrFAo4HK5xCySEEJ6PI1G0+YxA1EDXK1Wd/vcjYQQ0tu0l52i98AzHUQjhBDSUnvZKWqA82eKEEIIyV572Sl6gNOFn4QQ0jHtZaeoAc6fAtVR/kgc26rp4Cch5MzUXnaKGuCpVKrDAd4YiOLcpz7Bba9vQySe+aIfQgjpbdrLzrzvgdt0Sgwr0iMQTeDTfY5uqhkhhOSvvOiBJ5NJ4dLVjpg5pul2sP/ZXdvVVSKEkLzXXnaKGuDRaBRKpbLDn5t2VhEAYP3BBkQTNIxCCDmztJedogZ4JBLp1MQBpRYNhhfp4Y8ksOFAYzfUjBBC8ld72SlqgMfj8U7fyYzvhW88RAFOCDmztJedogZ4LBaDQqHo1GcnD7YCADYcTH9vXUII6W3ay84e0wMf198MvVKGww1B1LjS3weYEEJ6k7zogYfDYajV6k59Vi6VYMpQGwDg8wPUCyeEnDnay07RApy/qb3JZOr0Mi4a1jTb+EYaRiGEnCHSZadoAc5PS2Q0Gju9jLJCPQDghIduSUsIOTOky07RAtzj8QDAaQV4sbFpF6LWk3luS0II6Q3SZadoAd7Y2HT6n9Vq7fQyCvRKyCQcnMEYXdBDCDkjpMtO0QLc7Xa3W4lsSSUcLNqmU2kaA7EuqRchhOSzdNkpeg/cYrGc1nJKTE3DKCfcNA5OCOn90mWn6GPgZrP5tJZTatEAAJ0LTgg5I6TLTtECPBRqClytVntayxlgbQrwamfwtOtECCH5Ll12ihbg9fX1kMvlMBgMp7Wc/tamL1HtpB44IaT3S5edoga43W6HRHJ6RQ4qaArwI42BrqgWIYTktXTZKVqA19bWoqio6LSX0/e7g5jH6SAmIeQMkC47RQtwh8OB4uLi015OgV4JtVwKTygObyjeBTUjhJD8lS47RQvwhoYG2Gy2014Ox3HoTwcyCSFniHTZKUqAM8bgcDhgt9u7ZHn8OHgVjYMTQnqxTNkpSoB7vV7EYrEuC/DBBToAwCEHBTghpPfKlJ2iBLjD4QAAFBYWdsnyhtibAvxgPQU4IaT3ypSdogS4z+cDcHp3ImyOv63sgXp/lyyPEELyUabsFG0IJV0lOmpwgQ4yCYejrhBCsUSXLJMQQvJNpuwUtQeu1+u7ZHkKmQTDi/VgDHhx3eEuWSYhhOSbTNkpaoCf7mX0zT18ZTkkHLB43SG8uqGqy5ZLCCH5IlN2ijqEcjrzYZ5qylAbnpo9CgDwm//swyEHjYcTQnqXTNkpaoB3ZQ8cAG6a0A9zz+sHxoCn11R26bIJISTXMmWnKAEeCASgUCggl8u7fNn3Xj4UOqUMn+xzYOPBxi5fPiGE5Eqm7BQlwOPxeLeENwDY9SrcccFAAMC/dpzoljIIISQXMmWnKAEejUahUqm6bflXj2q60cvn+x1Ipli3lUMIIWLKlJ2iBHgwGIRGo+m25Q+169DPooEzGMOOY+5uK4cQQsSUKTtFCfBIJNKtPXCO43BpedO9Aj7Z5+i2cgghREyZslO0AFer1d1axuUjmu4VsLaiFozRMAohpOfLlJ2iBHgoFOr2AJ8w0AqTRo5qZwiHG+gmV4SQni9Tdvb4s1B4UgmHy8qbeuHv76rt1rIIIUQMeXEWCoDTnsw4G7PGlgAAVu84QcMohJBeIV12ijYjjxgmDbLCqlWg2hnCnpM+UcokhJDukik7ReuBi0EmlQjnhL/7LV3UQwjp3UQJcI7jkEqlxCgKc8b1AQCs/vYkXdRDCOnRMmWnKAEukUhEC/CzS03oZ9GgwR/FtmqXKGUSQkh3yJSdvS7AOY7D9NE0jEII6flOO8B/8pOf4J///Kfw7wULFmD58uUdqoRMJkMiId7UZzPGNJ2N8vFeB1I0jEII6aEyZacs0wJ27NgBi8WC66+/Hh6PB4sXL8bQoUPbfO++fftQWVkJiUQCpVIJk8mE8ePHix7gw4v06GNS44QnjG+PezCun1m0sgkhpKtkys6MPfDm09kbDAZceeWV+PLLL9t878qVK3Httddi9uzZuOqqqzBp0iT4fD7RA5zjOOHS+k/31YtWLiGEdKXTDvCKior/vlkiwQUXXACn09nme5PJZKvn+CuJ4vF4NvXtMvzNrd799iQCUZq5nhDS82TKzoxDKEeOHMG4ceOEfzPGcOLECQSDQWi12hbvLS8vx6xZs5BMJhGNRuF2u5FIJKBSqRCJRE7ja3TchIFW9DWrcdwdxl1vbsey286DXNqrTnsnhPRymbIzY4BrtVpUV1ejoqICn3zyCZYvX469e/di/Pjx2Lt3b4v3zp07F3Pnzm21DKVSiWg02leUp10AACAASURBVInqd55CJsHf75iA7730JTYdcuKPnxzAL6YNF7UOhBByOjJlZ8YA/973vof777+/1fMvvvhi1pVQKBSIxWJZv7+r9Ldq8dIPx+H6l7/EaxurcceUQTBrFaLXgxBCOiNTdmYcU3j88cfxzDPPYMWKFXA6nXj44YcxceJEXHTRRVlXQqPRIBwOZ/3+rjR+gAVTywoQjifx1lfHclIHQgjpjEzZmTHA+/XrhwceeAA33ngjLBYL9Ho9du3a1eYBy0yVEOtinlP9z5SmSY+Xba5GJJ59vQkhJJcyZWeHj+r98Ic/xCeffAKpVNqhSgAQ/UAmb+pQG8qLDWjwR/HON8dzUgdCCOmoTNnZ4QDv168fJk2a1KHP6PV6AIDf7+9ocV2C4zj85KLBAICXvziMRDI3ewKEENIRmbJTlPPqdDodACAQyN1UZ1ePKsZAmxY1rjA+3EMX9xBC8l+m7BQlwPlZlXN1IBNomnLttvMHAABe23SEZuwhhOS9TNkpSoDzk3LmMsABYM7ZfaBVSPH1UTceemcXDaUQQvJapuw8owJcr5Lj2RvGQC2XYuX241iy4UhO60MIIenkRYDzl9wHg0ExikvrypHF+PPcswEAyzYfoV44ISRvZcpOUQLcYDAAyN1ZKKe6rNyOwQVa1Pui+KzSkevqEEJImzJl5xnXAweaTiu8cXwpAODtr+m8cEJIfsqLHjh/Kky+BDgAzB7bBxIO+Hx/A9xB8e/TQgghmWTKTlEC3GQyQSKRwOHIn+EKu0GFyYNtiCVT+MXbO2kGe0JI3smUnaIEuEwmg81my6sAB4BHri6HUS3HJ/scmPfGdhxuyN2FRoQQcqpM2SnaDAc6nS5vDmLyRpQYsOTmc6GWS/FppQPfe2kzvGFxZw4ihJB00mWnaAGu1Wrzagycd95AC9Y9cBHGlprgCcXx1la65SwhJH+ky05RAzwUColVXIcUGVX42aVDAQDvfnsix7UhhJD/SpedogW4Xq/PuyGU5iYNtkIpk6Cyzg9nQNzp3wghpD3pslO0ADcajfB6vWIV12EquRTn9DcDALYeceW4NoQQ0iRddooW4AaDIa8DHAAmDrICALZWOXNcE0IIaZIuO0ULcLPZDI/HI1ZxnTJhoAUAsOFQI91ulhCSF9Jlp6inEYZCoZzNi5mNMaUmGFQyVDUEsXQj3amQEJJ76bJTtADnb0yeq3kxs6GSS/HM9WMAAP+3dj+q6MIeQkiOpctOUXvgQH7dD6Ut084qwpyz+yCWTNEEyISQnEuXnaIFuNXadICwoaFBrCI77dpxfQAAn+7Lr0v/CSFnnnTZKXqAu91usYrstPMGWqCWS1FZ50e9L3+HfAghvV+67BR9CCWXM9NnSymTYsKgpjNSttAphYSQHEqXnaJeiQnkz6w8mYwf0BTg26rpoh5CSO6ky07RAtxiaQrExsZGsYo8Led9d074pkNOOiecEJIz6bJTtAAvKCgA0DMOYgLA2aUmmDVyHGkM4nBDfp85QwjpvdJlp2gBrlAooNPp4HL1jCEJmVSCC8uaGu7z/XQ2CiEkN9Jlp2gBDjQNxveEg5i8S8oLAQAf7qnLcU0IIWey9rJT1ABXKBSIxXrOBMKXDLdDIZNgW7Ube07m9424CCG9V3vZKWqAq1SqvL6U/lQ6pQw3ndcPALDgrR3wRWi6NUKI+NrLTgrwDB66cjiGF+lR1RjEDS9/iRpXfs4qRAjpvfIiwHvaEAoAqBVSLLn5XAwq0KKyzo8fLt2KZIpOKySEiCcvhlBkMhkSiYSYRXaJUosG//rJ+Si1qHHUGcL6Az3jVEhCSO/QXnaKGuBSqRTJZFLMIruMUS3HDeeUAgD+s7s2x7UhhJxJ2stO0QM8nyd0yGTayCIAwKeVDiSSPfd7EEJ6lvayU9QA7+mG2nUYVKCFKxjDJ/vqc10dQsgZTtQAT6VS4DhOzCK7FMdxwmmFj/yrgs5IIYSIor3sFDXAk8kkpFKpmEV2udvOH4ipZQVwBWO4++9f01AKIaTbtZedFOAdJJVwWHzT2ehjUqPihA+vrK/KdZUIIb1cXgR4KpWCRNLzh90NKjl+973RAIA/fnKAhlIIId2qvewUNU3j8TjkcrmYRXabKUNtmHZWIeJJhs8q6W6FhJDu0152UoCfhsu+u1shnZFCCOlOeRHgiUSi1wW4hGuaN9NPN7oihHST9rJT1AAPh8NQqVRiFtmtzFoFzh1gQTzJ8Pl+uryeENI92stO0QNcrVaLWWS3u2IETfpACOle7WWnqAEei8WgUCjELLLbTTur6fL6T/bV0/3CCSHdor3sFC3AGWMIBoPQ6XRiFSmKUosGkwZZEYmn8O6OE7muDiGkl0mXnaIFeDgcRjKZhF6vF6tI0cyd0HR5/RtfHqV7hRNCulS67BQtwH0+HwDAYDCIVaRopp1ViD4mNQ46Ali5vSbX1SGE9CLpslO0APd4PAAAk8kkVpGiUcqkePiq4QCA362tRGMgmuMaEUJ6i3TZ2eEAr6ysxM6dOztcCa+3aVZ3o9HY4c/2BNNHF2PKEBs8oTh+8rdvEIn3zIkrCCH5JV12pg3wAwcOoKioCFKpVHiUl5dj7NixWLVqVYcqwe8G9NYA5zgOv79+DOx6Jb6qduFXqytyXSVCSC+QLjtl6T5oMBgwb948pFIpjBs3DmvXrsWSJUswefJkXHHFFa3ev2/fPlRWVkIikUCpVMJkMkGj0aCsrAzBYBAAoNVqu+I75aUiowqv3Toe05/fiH9/exK/nnkWdMq0TUwIIWmly8606VJUVIQnnngCALBmzRp8+OGHKC0txZo1a9o8pWXlypX49a9/3er5v/3tb4hEIgAAs9nc4S/Qk4zsY8R5Ayz4qtqFf397Ejd9d4YKIYR0htPpBNB2dmY9Bj537lwcO3YMx48fx6OPPopQqPUtVNubsNhms8HhaLpjX2FhYbZF9lhzJzRNfvzWV0fBGJ1WSAjpvHTZmfX+/WeffYZoNIqdO3finnvuwdGjR7F69eoW7ykvL8esWbOQTCYRjUbhdrsRCARgtVrh8XigVCp73aX0bblqZDGeeG8vKk748Pxnh/DTS4fmukqEkB4qXXZmDPDKykp88cUXmDNnDux2O8xmM6RSKYqLi1u9d+7cuZg7d26by1m6dGmvPAe8LSq5FL+ZMwrz3/oGz318ABIOmH8JhTghpON8Pl+72ZlxCKWiogIPPvggioqKoFQqMWLECNx8881YvHhxhyrR2NgIi8XSoc/0ZFePKsazN4wBADz/2SG4grEc14gQ0hOly86MAX7dddfB4/Hg7bffxqOPPoqKigq8+uqrHZ7b0uVywWq1dugzPd2cs/viomEFiCZSWLqR5s4khHRcuuzM6iAmx3G49tpr8atf/QojRozoVCWCwWCvPoWwPfz497JN1XD4IzmuDSGkp0mXnaJdSh8IBHrdnQizMa6fGZeVFyIYS+LJ9/flujqEkB4mXXaKFuBOp/OMGgNvbuGMEVDJJXhv50msowmQCSEdkC47Rb2Z1Zka4KUWDX5+WRkAYP5b32BrlTPHNSKE9BTpslOUAI/H44hEIr3yXuDZumPKQMwaW4JgLImbX/uKQpwQklGm7BQlwHv7nQizIZNK8NwNY/H98aWIJlK4++/foMbV+mpWQgjhZcpOUQL8TLiRVTakEg5PzR6JqWUFcAVj+J+/boOf5tEkhLQjU3aKEuD8jaxUKpUYxeU1mVSCxTedjcEFWhyoD+Cny3fQNGyEkDZlyk4K8BwwqOR47dbxMGvkWLe/AU+vodMLCSGt5UWA0xh4a/2tWrz0w3Mgk3BYsuEI3v76eK6rRAjJM3kxBt6b58M8HRMHWfH4rLMAAI+ursAJTzjHNSKE5JNM2UkHMXPsBxP645pRxQjHk3jivT25rg4hJI/kxUFMfjeAeuBt+9X0cmgVUny4px7v7TyZ6+oQQvJEpuwUJcD9fj8AnNEX8qRTbFTjl1eXAwAe+dduHHPS+eGEkMzZKUqA+3w+SCQSaDQaMYrrkX4woR8uH1EIfySBBSt2IJFM5bpKhJAcy5SdogS4y+WCyWSCRCLarVd6HI7j8PvrxqDYqMLOGg9eWHc411UihORYpuwUJVFDoRD1vrNg1Mjx7PX8LD4Hsa/Wl+MaEUJyKVN2inYzK7lcLkZRPd7kITb8aGJ/JFIMD769C3EaSiHkjJUpOynA89BDVw1HH5Mau094sXTjkVxXhxCSI3kR4IlEAjKZTIyiegWdUobfzBkJAHj+04Nw+GgqNkLORJmyk3rgeeqiYXZcVm5HMJbE/66uAGN0wytCzjR50QOPxWJQKBRiFNWrLJxxFvQqGT7eW49X1lchEk/mukqEEBFlyk4aQsljpRYNnrluNADg6TWVGPHYWjz5/l7qjRNyhsiLIZRkMgmpVCpGUb3OlSOL8ej0ERhk0yLFgKUbj2DFtppcV4sQIoJM2SlKgDPG6CKe0/A/UwbiswcuEs4RX/juHnx91J3jWhFCulum7BQtVTmOE6uoXut75/TFjyb2RyyZwh1/3YbqxmCuq0QI6WbpslO0AKdx266xcMYIXDysAO5QHLNf3ITff7gf4Rgd3CSkt0qXnRTgPYxMKsHzN43D5MFWeEJxLF53CPf+41tqX0J6qZwHuFQqRTJJvcSuolPK8Pc7JuAf8yZCr5Rh7Z46vLqBrtgkpLfJlJ2iBLhMJqMA72Icx2HCICuevaHpwObTayux/KtjOa4VIaQrZcpOUQJcoVAgGo2KUdQZ54qzivDAFWVIphh+uWo3fvuffYgmaGNJSG+QKTtFCXC1Wo1wmCbs7S7zLxmKRXNGQcIBr6yvwqzFm3DIEch1tQghpylTdooS4FqtVpick3SPmyb0w8q7JmGAVYPKOj9mv7AJn1XW57pahJDTkCk7RQlwjUZDPXARnDvAgg9+egGuGVWMQDSB//nrdrz0+WE6Q4WQHipTdooS4HK5HLFYTIyiznhapQyLbzob919eBsaA362txPTnN2LVN8fhi8RzXT1CSAdkyk5R7jClUCgowEXEcRwWXDoUZUV6PLq6AntO+nDfyp1QSCW4ZLgdt50/ABMGWXNdTUJIBpmyU9QAZ4zRJfUimnZWES4sK8A73xzHv789ia+qXVi7pw4f7q3DLZMG4NJyO/qaNRhg1dDfhZA8lCk7RQlwpVIJxhgSiQRN7CAylVyKH0zojx9M6I86bwRvfFmNl784jGWbq7FsczUAwKZTYsElQ/DDif0hlVCQE5IvMmWnKGPger0eAODz0SzruVRkVOHBK4dj+Z0Tcd5AC87tb4ZNp0RjIIqF/96DuX/ZguPuUK6rSQj5TqbsFKUHbrU2jbe63W7h/0nuTBhkxcq7JgFous/Ch3vq8ei7Ffiq2oWr/rQBz1w3GleOLM5xLQkhmbJTlB642WwGALhcLjGKIx3AcRyuHFmED38+FZePKIQ/ksDdf/8GyzbRvVUIybVM2SlKgBuNRgCA1+sVozjSCRatAn/50Tl48MphYAz49Xt7sfDdCrosn5AcypSdol2JCYCuxsxzHMfhJxcNwf9dNxpyKYe/fnkUV/9pA/614ziSKboYiBCxZcpO6oGTVm44txRv/3gy+ls1ONwQxL3/2IlZL2zE5/sdFOSEiChTdop6ELOxsVGM4kgXGFNqwsf3XojVO07gD58cQMUJH259fRtsOgUmDLLitskDcO4AS66rSUivlik7ReuBq1Qq1NbWilEc6SIKmQQ3jC/FZ/dfhF9MG4b+Vg0aAzF8sKsWN/5lC17dUJXrKhLSq2XKTlECnOM4FBcXo66uToziSBdTK6S45+Ih+PyBi/DJfVNx5wUDkUwxPPXBPty+bBt2HHPnuoqE9EqZslO0OTHNZjM8Ho9YxZFuwHEchtj1+N9rRuBP3x8LpUyCzyoduP7lL7FyW02uq0dIr5QuO0ULcIPBQAcxe5FZY/tg08OX4NbJA5BIMTz4zi78bm0lHeQkpIuly05RA9zv94tVHBGBTafEr2eehd/MGQmphMNLnx/Gbcu2wRWkO08S0lXSZadoAW61WuFwOMQqjojoBxP6483/OQ8WrQLrDzTg6j9twIqvjsFP9x8n5LSly07RAryoqAgOh4Nmh+mlJg+24f0FU3BOfzPqfBE8vGo3Ln32C2w61AhvOE5/d0I6KV12dijAI5EI3nvvPVRXV3e4EoWFhUgmk3A6nR3+LOkZSkxqrJg3Eb+/fgxG9zXC4Y/iB69uxZjHP8K0P67HW1uPwU3DK4R0SLrszPpCnqqqKsybNw+ffvopRo0ahZ07d3ZoEoDCwkIAQENDA2w2W9afIz2LXCrBdef0xeyxJfjzZ4fwl/WHkUoBB+oDeORfu/HouxWYPNiK84fYIOU4nDPAjHH9zLmuNiF5K112ZhXghw4dwoUXXgidTofbbrsNr7/+OuLxOBQKRYv37du3D5WVlZBIJFAqlTCZTNBoNCgrK4NOpwMABAKBrvhOJM/JpBLcd3kZfn7pUCRSDGsqavHP7cexpcqJDQcbseHgf68su2CoDQ9OG45RfY05rDEh+SlddmYV4OvXr0dBQQE2b96MjRs3YtmyZW32vleuXIlf//rXrZ5fuHAhLrnkEgA0qcOZRiLhoJBwmDW2D2aN7QN3MIbPKh3YddyDWJLhvZ0nvwv0jbhmdDHuv7wMgwp0ua42IXnDYDAAaDs7swrw22+/HbfffjtisRj+/Oc/48c//nGb0/skk23fetTj8aStBDlzmLUKfO+cvvjeOX0BAA9OG4aXvziM1zdX44NdtVhbUYfLyu24/fyBNPEyIeiCAOf9+9//xubNm7F8+fI2Xy8vL8esWbOQTCYRjUbhdrsRCASQTCah0WgA0C1lSUtmrQK/vLoct54/AH/65CDe/vo4PtxTjw/31GNqWQFmjy3BhWUFsOqUua4qITmRLjuzDvDGxkY8+uijuOOOO4R52k41d+5czJ07t83X+Gv56WIe0pZioxpPf2807ruiDH/fcgxLNx7B+gMNWH+gARIOGNfPjKGFepxdasLMsSVQyaW5rjIhouB74G1lZ9YB/uabb6K2tha/+93vOlUJPvQpwEk6dr0K915ehlsmD8DqHSewbr8DW6qc2H7Uje1H3Vj+1TH8ds0+TDurCOcPsWFYkR5D7boOnRFFSE+SLjuzCvCqqio88MADSKVSGD9+PCQSCUaNGoWlS5dmXQm1Wg0ACIVo1nOSmUWrwO1TBuL2KQPhi8TxdbUbRxqDWP3tCew67sWKbTVY8d0NtCYMtODR6SMwsg+dxUJ6n3TZmVWAm81mzJo1CzabDWPHjsWuXbuEqX6yJZFIoFKpaAycdJhBJcfFw+24GMBt5w9AZZ0fH++tx67jXmw/6sLWIy5Mf34jRvc14vvj++Ga0cUwqlsfZCekJ0qXnVkH+KpVq067IhqNBuFw+LSXQ85cHMehvNiA8uKmcUFPKIY/f3oI73xzHLuOe7Hr+G78+r09uHS4Hef0N2N4kQFlhTrYDaoc15yQzmsvO0WZUo2n0+noQh7SpUwaBR6bMQIPXjlMuFjoyyon1lTUYU3Ff2+Cf/4QK+ZNHYwRxQYU6OmMFtKztJedoga4VqulACfdQiWXYs7ZfTHn7L446Qnj00oH9p704WC9H3trfdh0yIlNh5ruJTGsUI+rRhXh6lHFdACU9AjtZaeoAS6XyxGP0y1GSfcqManxo4n9hX97QjG8sr4KXx1xobLWh/31fuyv9+OPnxzEIJsWl59ViMvKC3F2qQkyqWg36CQka+1lp6gBrlAoEIvR3eiIuEwaBR66cjgAIJZIYdOhRqypqMVHe+tR1RjEK19U4ZUvqmBQyXBBWQFmjC7BhIEWmDRy6p2TvNBedlIPnJxRFDJJ0xktw+1YlEzhq2oXPt5bj3WVDlQ7Q/hgVy0+2NU0A7hZI8egAh0G2bQYYNMiEE0gnkih1KLBhEEWDCvUU8ATUeRFD1wqlbZ7vxRCxCaTSjB5sA2TB9uwcMZZOOoM4sM9dfjP7jocrPfDHYrj66NufH3U3ebnbTolzioxwKpTwKRWYIhdhwE2DUYUG2DSKNr8DCGd0V52ihrgEomEZmYheau/VYt5Uwdj3tTBYIyhzhfBkcYgDjsCOOoMwaiWQyrlcKg+gPUHG9AYiOKLAw2tlsNxwOi+JpxdakJfsxqj+5owttQEhYzG10nntJedogZ4KpWCTCZqkYR0CsdxKDaqUWxUY/Lg1hOQpFIMNe4QKuv88IXjaAhEcdgRxOGGAPae9GFnjQc7azzC+7UKKYYXG3DUGYIvEodCKoFcysGqU2JMXxPG9Tehv0WLQDSOPiYNhhbq6H4vRNBedoqapslkEkolnYNLej6JhEN/qxb9ra2vSA7FEtha5cLhhgCqGoPYdsSFg45Ai6GYWCIFAHCH4jjkCOCdb463WIZcymHiICsmDLSgv1WLoYU6WLQKyCQSmOng6hmnvewUNcATiQT1wEmvp1HIhAOlPIcvgso6P/pbNSg0qBBNpBBPpnDCHcY3x9zYWePBSW8EeqUMx1whHG4ItJq5iKdXyVBeZBBC3aiWo0CvxLh+ZpRaNGJ+VSKS9rJT1DSNRqPUAydnJLtB1eJyfn54xKZTYkypqdX7XcEYPt/vQGWdH0cagzhQ70cwmkA0noI/ksBX1S58Ve1q9bk+JjUGFWgxxK5DgV6JRJJhTKkJEwZaaEimB2svO0UN8EgkApWK7klBSCYWrQLXjuvb6nnGGBoCUeyr9eOwIwBfJA5PKI7j7jC2HnHihCeME55wmz13g0qGYUV6DCvSQ6uQYXixHuMHWNDHpKYhmTzXXnaKGuChUEiYXYIQ0nEcx8GuV8GuV+HCsoIWryWSKVQ7QzjSGMRBhx/eUBwpxvBllRN7T/rgiySwrdqNbdUtT4s0aeQ4q8SAkX2MGF6kx6g+Rgyy6SCRUKjni/aykwKckF5CJpVgiF2HIXYdLh9R2OK1VIrBGYyh4qQX1Y1B+CMJfFvjwTfH3PCE4i3uFQM0jbMPK9RjTKkJU4bYMH6gBTolHb/KlbwI8FgsBoWCLnAgRGwSCYcCvRIXD7MDw/77PGMMtd4Idp/wYu9JH/bX+bGjxo16X1SYBWnpxiOQcMDYUhMuHmbHAJsWVq0CdoMSJSY1NAoK9u7WXnbSQUxCzmAcx6HEpEaJSY1pZxUB+G6c3R9FZZ0fW4809cx3n/Dim2MefHPM02oZhQYlzioxCkMwpWYN+lk0MGpoUo2ukvODmIlEAvF4nIZQCMlzHMcJZ81MLSvAL6YBgWgCGw82YusRJ+q8EbiCMdT7IjjpiaDeF0W9z4HPKh0tltPXrMbkwVYUG9XoY1LjvIEW9LdqwHEcGGPCqZQ6pYwOoqaRLjtFC3B+OqCOTsVGCMk9nVKGK0cW4cqRRS2eT6YYalwh7D7hRcVJLw47AjjuDuOoM4Tj7jBWbm95gZJVq4BRI8cJdxjR7y5mkks5lFo0wrntQ+w6DLI13VeGsaaNR4FOKRxUjSdTOOYK4YQ7DItWAbteCblUghOeMJzBGGw6BaQSDvEEQzCWQDLFkEgxJFMpmDQKFBpUSCYZ4qkUGGMoMqqRYgyN/igAQCaRQCblEEukkGQMEo6DLxxHIsVg0Spg0ylE3eiky07RAtzlajpn1Ww2i1UkIaSbSSUcBnx3t8YZY0qE51Mphj0nfdh+1AVXMIaD9QF8Ve2CMxiDM9h0W1SlTAIJxyEcT6KqIYiqhiCwu+1ytAopiowqcByHo84g4snc3lNJJZegQK9EX5MGfcxNkw4nUwz+SAIHHX6EYklIOQ4lJhUGWLUYWqjHTRP6dWqu1nTZKXqA22yt7ytBCOldJBIOo/oaMaqvUXiOMYbj7jCCsQT6mjXCWS3hWBKHGwKorPPjoMOPw44gjjQGUOMKQyJpurLVFYzhcMN/J/Xta1ajr1kNVzAGVzCOaDyJYpMKNp0SrmAMyRSDXCqBVikVetQSjoMzGEWjPwaphBNuLnbSE4aE42A3NI0xJ5IMyRSDQiaBVMIhxRj0KhmkHAdnMIbGQBSReAo1rjBqXOnn+K3zRYTjBolkCgsuHdrhtkyXnaIFuM/nAwAYDAaxiiSE5BGO49q81F+tkGJkn6aDoM0xxoRhCmcgCncohmgihf5WbZee0sjf5S/bIRHGGALRBBoDMRxzhVDrCUMi4SDlmjYKw4r0MKnliCZSOO4O46ZXt4AxwNCJ3jeQPjtFC3Cv1wsAMBqNGd5JCCEtA9WqU8Kq654z2Do6ls1xHPQqOfQqOQba0h/TK7VosORH52JLlRPn9O/c8HG67BQtwN3upqu/aAycEHImuWxEIS475cKqjkiXnaLdYZ6fUVmn04lVJCGE9HjpslO0AA+Hmwb71Wq1WEUSQkiPly47RQtwr9cLqVRKF/IQQkgHpMtO0QLc7/dDr6dZvAkhpCPSZaeoPXCTqfWN6wkhhLTv2WefhcPhaPM1jok4TXwymYRU2jNnBWGMwev1wul0wuv1IhgMwuv1wu12w+l0wu/3IxqNIhaLIRaLIR6PIxQKIRgMIhwOIxaLIZFIIJlMtlgux3GQSqWQyWRQKBSQy+WQyWSQy+WQy+XQaDSwWCwwGAzQ6/UwGo3QarUwmUwwGo1QqVRQqVTQarUwGo2Qy3vnDYQSiQQ8Hg8CgQCCwSB8Pp/QtuFwGJFIBIFAAH6/H6FQSHjEYjFEo1FEIhHE43EkEgnhkUqlkPrucmrgv6eT8e3evG2VSiXkcjl0Oh2MRiOMRiMMBgMMBoPw/3a7HUajscfuZfr9frhcLgSDQeERCoXg9/vh9/uF9uX/n2/TSCSCaDSKeDyOWCzWYh3nOE5YtxUKBdRqNfR6vfBo3n4mkwkmk0n4f7PZ3CvW52g0ipMnT8LtdsPlcqG+vl5YfyORiLCuRqNRYZ3m19VkMolUKoXRo0fjI+AYuQAADxVJREFUmWeeabVs0U4j/NnPfoaKigqo1WqYTCZYLBYhkNRqNXQ6Hcxms/DHtFgssFgs0Gq1XTaPZiqVQjgcht/vh8/nQygUgs/ng8/nQyAQQH19Perr61FXVwen0ym85na7UVtbi0gkknb5HMcJKyq/smq1WqjVaiiVSkilUkilUnAcJ9zQJ5lMIhqNIpFICMHP37yG3wh4PB6kUqmsvqNKpYLJZILVaoVOp4NWq4XFYoHNZhN+GHa7HVarFVqtVvgB8T8ctbrrZ2eJxWJoaGiAy+USfvxOpxNOp1MIgkAgALfbDZ/PB6/XC7/fL4RIIBBAY2Nj1m0ANB3wUavVUCgUUCqVUKlUwsaRf0gkEuEBNG2k+XWkvr5e2DCEQiEhrGKxWNpyFQoF7HY7CgoKYLfbUVxcjMLCQhQWFkKj0cBkMsFms8FsNsNms8FkMkGn0wl1OF2MMUSjUaHzwIcw3/mora1FXV2d8N+6ujq4XC7hb5ENpVIJnU4HtVoNmUwGlUolbOAUCoWwjgNNnbZIJCJ0bCKRiPD74w/OpaPRaKDT6aDX64U2tVqtsFgs0Gg0KCgogM1mE9Z1o9EIs9ksbAy6ol0ZY4jFYgiFQggEAvD5fGhoaIDb7Rb+zX8nvlNXW1uLhoYGOBwONDQ0pF0+P76tVCqFvGi+rkqlUoRCoTY/K1oP/Gc/+xm2b9+OSCQCl8sFj8cDv9/fqkfaFrlcDqVSCYVCAY1GI/SOlEql8AUlEglSqRSSyaTwQ4vH40IA8D/CTKRSKex2O+x2u7CBMZlMKCoqQnFxMWw2m9ALNhqNsFgsMJvNMBgMkMm65wY3qVRK6Al5PB4Eg0F4PB54vV5EIhFEIhFhj4DvRblcLqG36nQ64XK54PP5EI1GM35/rVYrbID4Hym/RyCRSIQNEf/jSCaTSCaTwkaIr1MsFkMgEMgqGPhw43u3er0eGo0GWq0Wer1e+JtotVrhOX5l5x/8D12lUnVZIJ4qHo/D5/PB4/EIP1yv1wuv14v6+no4HA44HA40NjYKIelwOBCPx9tdJsdxwsaTD0G5XC6s43wgSiQScBwn7DnEYjGEw2EhWPjeW6aftEQigd1uR0lJCYqKimCz2WCxWFBSUgKr1Sq0u1arhUajEfb+dDoddDpdl/WKk8lkiw22x+MR2tXj8cDtdgs54ff7hXZtaGiAx+NpN9RObVetViu0K58jfEDyIwLN1+FoNIpoNIpwOCzs9WUTkzKZTMiLwsJCoW379OmDPn36CBvuwsJCGI1GIcfkcnmnc0PUIZRTMcYQCoUQDoeFHpjX64XP50NjYyPcbrfQg+CHJ/jdDX63jd/N4C+7lUqlLX4E/ErH94Y1Go2w+8b3QA0GA3Q6HQoKCmC1WnvsLjCPMQa32w2dTtfqJvChUAgOh0NoWz58mgdSIBAQwoHvefIPfiPJtzkAIdT5XWV+6EGhUECn08FisQg9JT4IzGYzCgoKoNVquzVwsxGPx4V2ksvlGXvZHZVKpYRdZn43mt8Dad7+/K4z3/ng13G+rfkHH+ZKpbLFxotfv/l1nf83v55brVZhQ3g67Z1IJDB27FhhY1FVVdWFrZW9VCqFxsZGYe+h+bCmx+MROorBYFBYf/mOBb+ny+/VNV+HlUollEql0KnQ6XRQqVRCdvBtabFYoNPphA1cur3XVCqFP/3pT/B6vXA4HHjxxRe7pA1yGuDZiMfj+Prrr4Xd4PLy8lxXKe+53W5YLBYATZffejytb8JP/qt5e2m12qyHEs5U1F4d111tlvdzIe3btw+TJk0CAAwfPhz79u3LcY3yX01NjfD/xcXFOaxJz9C8vUpLS3NYk56B2qvjuqvNcrffmqXm41x0GX52qM06htqrY6i9Oq672izvA7z57j/dCCs71GYdQ+3VMdReHdddbZb3Ad78rAmaEDk71GYdQ+3VMdReHdddbZb3Ad783N+eehGQ2KjNOobaq2OovTquu9os7w9iDh8+HI8//jiSySSGDx+e6+r0CNRmHUPt1THUXh3XXW2W96cREkIIaVveD6EQQghpGwU4IYT0UD0iwN1uN/bv35/ravRotf/f3pnFNLV9YfxrmUyRCgg0NQpeBAWN1gk0jtE4BpEYqRAnsARRExWNA0ExUeMUg+KAIqgxGjUIGnHAB31weDAqUdQYh0iEUBmEtrFIoYX2uw8mJ/SiN/r/XzlVzi/pw9lD93dW1lnndHeftWtrUVVVJbYMUaiqqsL27dvx/v37TnUWiwUvXrz4bt+GhgZUVFT8Snkux8ePH7F9+3a8efPmf+pfV1eHysrK/1bUb4DdbkdZWRmMRmOnutbWVpSXl383p4rRaMS7d+9+flC6MA6Hg/v27aNSqSQAJiUl0WaziS3LZaipqWFJSQlLSkpYVFTEtLQ0hoeHMzk5mQ0NDSRJm83G9PR0enl50c3NjVlZWbTb7SIr71pycnIIgAMGDKDRaBTKz5w5Q7VaTQCcMWMGzWazUGe327l161YqFArKZDKuWbOG7e3tYsjvcvLz8wmAwcHB/PTpE0mytrZW8LXi4mKmpaVx4MCBTEpKEtrYbDauW7dO8LUtW7Z0G1+rr6/nggULCIATJkygw+EQ6goLC9mvXz8C4MSJE5180OFwcNeuXfTx8SEApqSk/FSMc+kAXlBQQLlcTn9/f65Zs4bR0dGcOnUq29raxJbmEixbtowAhE9gYCDVajXVajXz8/NJkps2baKnpyeVSiV37tzJ/v37MyUlxcnB/nRqa2upUCgIgHv27CFJ3rlzhwAYEBDAxMRExsTEMCIigl++fCFJ7t27l+7u7vT19WVmZiYjIyM5f/78bmE3g8EgBJSsrCySpE6n+66v5eXlkSQzMjLo4eFBpVLJHTt28K+//qJOp/vjbabX6xkeHs6goCDOmTOHAISb/cOHDymTyRgYGMi4uDhqtVqGhITQZDKRJHNzcymXy+nn58f169dz5MiRnDlz5g8/LLhsALfb7Rw9ejT37NnD9vZ2Wq1WmkwmDhw4kPv37xdbnksQHR3NPn368M6dO3z+/HmnC6W5uZkqlYpFRUVsaWmhw+FgZWUlfX19efXqVZFUi0NMTAwBMD09nSQZHx/P1NRUOhwOtrS00Gq1ctKkSVy5ciVtNhvDwsKYn59Pq9XK9vZ21tXVsU+fPjx9+rTIZ9I1aLVaAuDy5ctJkmPHjhV8rby8vJOvWSwWqtVqXrp0SfC1qqoq+vn58cqVK2KcQpeRk5PD8PBwmkwm3r592ymA63Q6JiYmkiRbWlrY1tbG2bNnc/HixWxvb6dGo2F2djbb2tpos9loMBgYGhrKQ4cO/dDYLjsH/vLlS7x69QpJSUlwc3ODp6cnfH19MWTIEJjNZrHluQQVFRVQqVRobm6Gv79/p1SWpaWlcHd3R2xsLHr06AGZTIaQkBD07du329lQq9UCAF68eAGDwYCSkhKsWLECMplMSH0bFRUFs9mM+/fvw2AwICEhQdigQKVSISwsrNvYLT4+HsDX6xD46mtBQUFobm7+ZsrlW7duQSaTYe7cuYKvBQcHo1+/fn+8zVavXo2ysjL07NkTBQUFWLRoEdzc3NDU1ITCwkKsWrUKwNfNVtzd3TFmzBiYzWY8ffoU79+/x5IlS4Sc+/7+/oiMjPxxm/2y29L/SX5+PocPH+5UptfrqVKp+PjxY5FUuRbJycnCT1oPDw8eOHDAac5x48aN1Gq1Tn2ePn1Kb29vNjY2drVcUbl79y4BMDQ0lLdv32avXr2c6puamhgREcHCwkLu3r2b06ZNc6p/+/Yte/XqxYqKiq6ULRqPHz8mAKrVapLOUyju7u7Mzs528rXNmzdz/vz5Tt/x7Nkz9uzZU5gj/9O5evUqlUqlMMf96NEjymQypzatra0cMWIET548yaNHjzI6OtqpvrKykgEBASwvL/+hMV36TcyOr59arVakpqZi1qxZiIqKElGV65CTk4Phw4fDYDCgpqYGGRkZ0Ov1yM7OFtp0tKHRaIROp0NGRgZ69+4thmSXoaNdHA4H1q9fD5VKhfj4eOzbt8+p/suXL9DpdEhJSUFoaKgYckXn4MGD0Gg0aGxsRE1NDTIzM1FdXY2DBw8KbTrazGQyQafTYdOmTQgMDBRDcpdiNpuxa9cuJCYmOiWrYodNOEgiMzMTcrkcSUlJOHHihJPNWltbkZqainnz5kGj0fzYwP/BjeeX8OjRI8rlcmZnZ7O4uJjjx49nQkICm5ubxZbmskyfPp1jxowRjs+fP0+FQsHTp0/z3LlzHDp0KNetW9dtVlN0pOMTuF6vp4eHBzMyMnjt2jXGxsZyypQprK+vJ0mWlpbS09OTx44d48WLFzlq1CimpqbSarWKfBZdxz+fwP/JzJkzGRUVJRxfuHCBCoWCp06d4rlz5zhs2DCmp6d3G1/Ly8ujr6+v0wKLhoYGent7c+3atbx+/Tq1Wi3HjRvH6upqkuSDBw/o5ubGnJwcFhUVcezYsVy0aBEtFssPj+uyAZwkjx8/zt69e9Pf35+5ubliy3FpLBYLIyIiOHnyZKHMbrczKyuLCoWCISEhvHz5sngCRaZjACfJy5cvMyQkhAqFotPSyo7LV9VqNc+ePSuWbNH4twBusVgYGRnJiRMnCmV2u53btm2jQqFgcHAwi4uLu1KuqHz48IFeXl6Uy+WMjY1lXFwct27dSpK8efMmw8LC6OXlxQ0bNnRaQXf48GH6+fkxICCAJ06c+OmxXT4XSlNTE0hCqVSKLcWleP36NZKTkwEAgwYNwpMnT/Du3Ttcu3YNMTExTm0bGxvh4+PTrVN/6vV65OXlQaPRCH9oWq1WNDU1ISAg4Jt9TCaTsDdid6O+vh5HjhzB4MGDMWrUKCxduhTAV18rKyvDmzdvUFJSgtjYWKd+3dHX6urqEBcXh6CgIGg0Gjx//hx+fn44e/YsgK/bQhqNRqhUqm/2//z5M+RyOXx8fH56bJcP4BLfxmazoaCgADdv3sTdu3fh7e2NI0eOIDExUWxpEn8YNpsNJ0+exI0bN3Dv3j0oFAocOnQICxcuFFtat0cK4H8AdrsdgJSbWeLXI/maayEFcAkJCYnfFJd9kUdCQkJC4t+RAriEhITEb4oUwCUkJCR+U6QALiEhIfGbIgVwCQkJid+UvwGy4wQb78dc1wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HXCy84vvgBYo",
        "colab_type": "text"
      },
      "source": [
        "## Store the weights to a safe place :)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "D-zrHcxhaSUh",
        "colab": {}
      },
      "source": [
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "\n",
        "PROJECT_ID = \"fast-ai-exploration\" \n",
        "!gcloud config set project $PROJECT_ID"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_OyIyB-kkHuN",
        "outputId": "5cbe7bc3-efc9-45a3-fcae-b59d58527d61",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import datetime\n",
        "filename = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") + \"resnet_simclr.h5\"\n",
        "filename"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'20200509-042927resnet_simclr.h5'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qNzIZ0e4kMGk",
        "outputId": "73093c87-1e6a-42af-9c4b-bfe09e5d6ebc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "#!gsutil mb gs://resnet_simclr_imagenet\n",
        "resnet_simclr.save_weights(filename)\n",
        "!gsutil -m cp 20200508-134915resnet_simclr.h5 gs://resnet_simclr_imagenet/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Copying file://20200508-134915resnet_simclr.h5 [Content-Type=application/octet-stream]...\n",
            "\\\n",
            "Operation completed over 1 objects/92.4 MiB.                                     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vUrvFEpesI4Q",
        "colab_type": "text"
      },
      "source": [
        "## In case saving to a GCS bucket is not possible"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CdkBEeB5sMU4",
        "colab_type": "code",
        "outputId": "583ce5ba-9a3b-45bf-cac1-e5e5d931c669",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "resnet_simclr.save_weights(filename)\n",
        "wandb.save(filename)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/content/wandb/run-20200509_033924-simclr-learning/20200509-042927resnet_simclr.h5']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    }
  ]
}