variational_autoencoder.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "variational_autoencoder.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "2r3gdX0b20rA",
        "colab_type": "code",
        "outputId": "9af187e9-1662-447f-9647-110e6eb0ea99",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 63
        }
      },
      "source": [
        "import tensorflow.keras as keras"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "isRU_R6D5wrL",
        "colab_type": "code",
        "outputId": "d94da01a-3e52-4e3a-8299-58795dc7c887",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#from keras.engine.input_layer import Input\n",
        "from keras.models import Model\n",
        "from keras.layers import Flatten, Conv2D, Conv2DTranspose, Dense, MaxPool2D, Dropout, Activation, BatchNormalization, Reshape, Input, Lambda\n",
        "from keras.layers.advanced_activations import LeakyReLU\n",
        "from keras.optimizers import Adam\n",
        "import keras.backend as K\n",
        "from keras.losses import mean_squared_error\n",
        "\n",
        "from keras.callbacks import Callback, ReduceLROnPlateau, ModelCheckpoint , LearningRateScheduler\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8TYFfWmVYh6J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class CustomCallback(Callback):\n",
        "\n",
        "  def LR_scheduler(self):\n",
        "    return ReduceLROnPlateau(monitor='loss', factor=0.1, patience=1, verbose=1)\n",
        "  \n",
        "  def step_decay_schedule(initial_lr, decay_factor, step_size):\n",
        "    '''\n",
        "    Wrapper function to create a LearningRateScheduler with step decay schedule.\n",
        "    '''\n",
        "    def schedule(epoch):\n",
        "        new_lr = initial_lr * (decay_factor ** np.floor(epoch/step_size))\n",
        "        \n",
        "        return new_lr\n",
        "\n",
        "    return LearningRateScheduler(schedule)\n",
        "\n",
        "def step_decay_schedule(initial_lr, decay_factor=0.5, step_size=1):\n",
        "  '''\n",
        "  Wrapper function to create a LearningRateScheduler with step decay schedule.\n",
        "  '''\n",
        "  def schedule(epoch):\n",
        "      new_lr = initial_lr * (decay_factor ** np.floor(epoch/step_size))\n",
        "      \n",
        "      return new_lr\n",
        "\n",
        "  return LearningRateScheduler(schedule, verbose=1) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TwA8XbBf2xti",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Variational_AutoEncoder:\n",
        "\n",
        "  def __init__(self, input_dim, n_conv_layers, conv_layers_filters, conv_layers_kernels, conv_layers_strides, conv_transpose_layers_filters, conv_transpose_layers_kernels, conv_transpose_layers_strides, z_dim, learning_rate, r_loss_factor, x, y, epochs):\n",
        "    self.input_dim = input_dim\n",
        "    self.n_conv_layers = n_conv_layers\n",
        "    self.conv_layers_filters = conv_layers_filters\n",
        "    self.conv_layers_kernels = conv_layers_kernels\n",
        "    self.conv_layers_strides = conv_layers_strides\n",
        "    self.conv_transpose_layers_filters = conv_transpose_layers_filters\n",
        "    self.conv_transpose_layers_kernels = conv_transpose_layers_kernels\n",
        "    self.conv_transpose_layers_strides = conv_transpose_layers_strides\n",
        "    self.z_dim = z_dim\n",
        "    self.learning_rate = learning_rate\n",
        "    self.r_loss_factor = r_loss_factor\n",
        "    self.x = x\n",
        "    self.y = y\n",
        "    self.epochs = epochs\n",
        "\n",
        "  def Encoder(self):\n",
        "\n",
        "    self.encoder_input = Input(shape=(self.input_dim), name='Encoder_Input')\n",
        "    x = self.encoder_input\n",
        "\n",
        "    for i in range(self.n_conv_layers):\n",
        "\n",
        "      conv_layer = Conv2D(filters= self.conv_layers_filters[i],\n",
        "                          kernel_size= self.conv_layers_kernels[i],\n",
        "                          strides= self.conv_layers_strides[i],\n",
        "                          padding='same',\n",
        "                          name = 'Conv_Layer_'+str(i))\n",
        "      \n",
        "      x = conv_layer(x)\n",
        "      x = Dropout(0.25)(x)\n",
        "      x = LeakyReLU()(x)\n",
        "      x = BatchNormalization()(x)\n",
        "    # int_shape in used because using Keras.shape() will output a Tensor but int_shape outputs a tuple\n",
        "    \n",
        "    self.shape_before_flattening = K.int_shape(x)[1:]\n",
        "    x = Flatten()(x)\n",
        "\n",
        "    self.mu = Dense(self.z_dim, name='mu')(x)\n",
        "    self.log_var = Dense(self.z_dim, name='log_var')(x)\n",
        "\n",
        "    #self.encoder_mu_log_var = Model(encoder_input, (self.mu, self.log_var))\n",
        "\n",
        "    def sampling(args):\n",
        "      mu, log_var = args\n",
        "\n",
        "      epsilon = K.random_normal(K.shape(mu), mean=0.0, stddev=1.0)\n",
        "      return mu + K.exp(log_var/2)*epsilon\n",
        "\n",
        "    self.encoder_output = Lambda(sampling, name='encoder_output') ([self.mu, self.log_var])\n",
        "    self.encoder = Model(self.encoder_input, self.encoder_output)\n",
        "  \n",
        "  def Decoder(self,flag=0):\n",
        "    #if flag==1:\n",
        "    self.decoder_input = Input(shape=(self.z_dim,), name='Decoder_Input')\n",
        "    x = Dense(np.prod(self.shape_before_flattening))(self.decoder_input)\n",
        "    x = Reshape(self.shape_before_flattening)(x)\n",
        "\n",
        "    #else:\n",
        "    #  x = Dense(np.prod(self.shape_before_flattening))(self.encoder_output)\n",
        "    #  x = Reshape(self.shape_before_flattening)(x)\n",
        "                                                         \n",
        "    for i in range(self.n_conv_layers):\n",
        "\n",
        "      conv_transpose_layers = Conv2DTranspose(filters = self.conv_transpose_layers_filters[i],\n",
        "                                              kernel_size = self.conv_transpose_layers_kernels[i],\n",
        "                                              strides = self.conv_transpose_layers_strides[i],\n",
        "                                              padding='same',\n",
        "                                              name = 'conv_transpose_layer_'+str(i))\n",
        "      \n",
        "      x = conv_transpose_layers(x)\n",
        "      x = Dropout(0.25)(x)\n",
        "      \n",
        "\n",
        "      if i<self.n_conv_layers-1:\n",
        "        x = LeakyReLU()(x)\n",
        "      else:\n",
        "        x = Activation(activation='sigmoid')(x)\n",
        "      x = BatchNormalization()(x)\n",
        "\n",
        "    self.decoder_output = x\n",
        "    self.decoder = Model(self.decoder_input, self.decoder_output)\n",
        "\n",
        "\n",
        "  def Model_combine(self):\n",
        "    \n",
        "    #model_input = self.Encoder()\n",
        "    #model_output = self.Decoder()\n",
        "    self.Encoder()\n",
        "    self.Decoder()\n",
        "    model_input = self.encoder_input\n",
        "    model_output = self.decoder(self.encoder_output)\n",
        "    #self.model = Model(self.encoder_input, self.decoder_output)\n",
        "\n",
        "    self.model = Model(model_input, model_output)\n",
        "\n",
        "    return self.model\n",
        "\n",
        "  #def r_loss(y_true, y_pred):\n",
        "  #  return K.mean(K.square(y_true,y_pred), axis=[1,2,3])\n",
        "\n",
        "  def get_model(self):\n",
        "\n",
        "    self.model = self.Model_combine()\n",
        "\n",
        "    optimizer = Adam(lr=self.learning_rate)\n",
        "\n",
        "    def vae_r_loss(y_true,y_pred):\n",
        "      r_loss = K.mean(K.square(y_true - y_pred), axis = [1,2,3])\n",
        "      return self.r_loss_factor * r_loss\n",
        "\n",
        "    def vae_kl_loss(y_true, y_pred):\n",
        "\n",
        "      kl_loss = -0.5* K.sum(1+ self.log_var -K.pow(self.mu,2) - K.exp(self.log_var), axis=1)\n",
        "      return kl_loss\n",
        "\n",
        "    def vae_loss(y_true, y_pred):\n",
        "      r_loss = vae_r_loss(y_true, y_pred)\n",
        "      kl_loss = vae_kl_loss(y_true, y_pred)\n",
        "      return r_loss + kl_loss\n",
        "\n",
        "    self.model.compile(optimizer = optimizer, loss = vae_loss, metrics=[vae_r_loss, vae_kl_loss])\n",
        "    print(self.model.summary())\n",
        "\n",
        "    \n",
        "\n",
        "  def train(self):\n",
        "\n",
        "       \n",
        "\n",
        "    initial_lr = self.learning_rate\n",
        "    lr_scheduler = step_decay_schedule(initial_lr, 0.1, 5)\n",
        "\n",
        "    self.model.fit(self.x, self.y, batch_size=500, shuffle=True, epochs=self.epochs, callbacks=[lr_scheduler])\n",
        "    return self.model, self.encoder, self.decoder\n",
        "\n",
        "  def Encoder_model(self):\n",
        "    #self.Encoder()\n",
        "    #self.encoder = Model(self.encoder_input, self.encoder_output)\n",
        "    #optimizer = Adam(lr=self.learning_rate)\n",
        "    \n",
        "    #self.encoder.compile(optimizer = optimizer, loss = 'mean_squared_error')\n",
        "    print(self.encoder.summary())\n",
        "\n",
        "    return self.encoder\n",
        "\n",
        "  def Decoder_model(self):\n",
        "    #self.Encoder()\n",
        "    #self.Decoder(flag=1)\n",
        "    #self.decoder = Model(self.decoder_input, self.decoder_output)\n",
        "    #optimizer = Adam(lr=self.learning_rate)\n",
        "    \n",
        "    #self.decoder.compile(optimizer = optimizer, loss = 'mean_squared_error')\n",
        "    print(self.decoder.summary())\n",
        "    return self.decoder\n",
        "\n",
        "   \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v6X7R1XIVqoe",
        "colab_type": "code",
        "outputId": "c3742c41-f9c5-4b1a-9279-8ac34679f307",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 498
        }
      },
      "source": [
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "mnist = input_data.read_data_sets('MNIST_data', validation_size=0)\n",
        "\n",
        "(x_train, y_train), (x_test, y_test) = (mnist.train.images, mnist.train.labels), (mnist.test.images, mnist.test.labels)\n",
        "train_images = x_train.reshape(60000,28,28,1)\n",
        "test_images = x_test.reshape(10000,28,28,1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-5-de9e64603ff5>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please write your own downloading logic.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use urllib or similar directly.\n",
            "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
            "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
            "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
            "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
            "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
            "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UjCrc_js9lAj",
        "colab_type": "code",
        "outputId": "53cf3284-cd04-4e14-9db1-31f3fad4af53",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "VAE = Variational_AutoEncoder(input_dim = (28,28,1)\n",
        "    , n_conv_layers = 4  \n",
        "    , conv_layers_filters = [32,64,64, 64]\n",
        "    , conv_layers_kernels = [3,3,3,3]\n",
        "    , conv_layers_strides = [1,2,2,1]\n",
        "    , conv_transpose_layers_filters = [64,64,32,1]\n",
        "    , conv_transpose_layers_kernels = [3,3,3,3]\n",
        "    , conv_transpose_layers_strides = [1,2,2,1]\n",
        "    , z_dim = 2\n",
        "    , learning_rate = 0.005\n",
        "    , r_loss_factor = 1000\n",
        "    , x = train_images\n",
        "    , y = train_images\n",
        "    , epochs = 30\n",
        ")\n",
        "\n",
        "model = VAE.get_model()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:203: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2041: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4409: The name tf.random_normal is deprecated. Please use tf.random.normal instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "Encoder_Input (InputLayer)      (None, 28, 28, 1)    0                                            \n",
            "__________________________________________________________________________________________________\n",
            "Conv_Layer_0 (Conv2D)           (None, 28, 28, 32)   320         Encoder_Input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 28, 28, 32)   0           Conv_Layer_0[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_1 (LeakyReLU)       (None, 28, 28, 32)   0           dropout_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 28, 28, 32)   128         leaky_re_lu_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "Conv_Layer_1 (Conv2D)           (None, 14, 14, 64)   18496       batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "dropout_2 (Dropout)             (None, 14, 14, 64)   0           Conv_Layer_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_2 (LeakyReLU)       (None, 14, 14, 64)   0           dropout_2[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 14, 14, 64)   256         leaky_re_lu_2[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "Conv_Layer_2 (Conv2D)           (None, 7, 7, 64)     36928       batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "dropout_3 (Dropout)             (None, 7, 7, 64)     0           Conv_Layer_2[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_3 (LeakyReLU)       (None, 7, 7, 64)     0           dropout_3[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 7, 7, 64)     256         leaky_re_lu_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "Conv_Layer_3 (Conv2D)           (None, 7, 7, 64)     36928       batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "dropout_4 (Dropout)             (None, 7, 7, 64)     0           Conv_Layer_3[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_4 (LeakyReLU)       (None, 7, 7, 64)     0           dropout_4[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 7, 7, 64)     256         leaky_re_lu_4[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "flatten_1 (Flatten)             (None, 3136)         0           batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "mu (Dense)                      (None, 2)            6274        flatten_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "log_var (Dense)                 (None, 2)            6274        flatten_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "encoder_output (Lambda)         (None, 2)            0           mu[0][0]                         \n",
            "                                                                 log_var[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "model_2 (Model)                 (None, 28, 28, 1)    102661      encoder_output[0][0]             \n",
            "==================================================================================================\n",
            "Total params: 208,777\n",
            "Trainable params: 208,007\n",
            "Non-trainable params: 770\n",
            "__________________________________________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jLf9NFTZ-CTE",
        "colab_type": "code",
        "outputId": "b1e6900e-009b-4d23-fd0c-34196a106cc5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "model, encoder, decoder = VAE.train()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/math_grad.py:1375: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n",
            "Epoch 1/30\n",
            "\n",
            "Epoch 00001: LearningRateScheduler setting learning rate to 0.005.\n",
            "60000/60000 [==============================] - 12s 201us/step - loss: 440.9876 - vae_r_loss: 432.2273 - vae_kl_loss: 8.7603\n",
            "Epoch 2/30\n",
            "\n",
            "Epoch 00002: LearningRateScheduler setting learning rate to 0.005.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 112.9985 - vae_r_loss: 109.2038 - vae_kl_loss: 3.7947\n",
            "Epoch 3/30\n",
            "\n",
            "Epoch 00003: LearningRateScheduler setting learning rate to 0.005.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 70.8371 - vae_r_loss: 67.7578 - vae_kl_loss: 3.0793\n",
            "Epoch 4/30\n",
            "\n",
            "Epoch 00004: LearningRateScheduler setting learning rate to 0.005.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 68.1521 - vae_r_loss: 65.2561 - vae_kl_loss: 2.8960\n",
            "Epoch 5/30\n",
            "\n",
            "Epoch 00005: LearningRateScheduler setting learning rate to 0.005.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 67.7255 - vae_r_loss: 64.8172 - vae_kl_loss: 2.9083\n",
            "Epoch 6/30\n",
            "\n",
            "Epoch 00006: LearningRateScheduler setting learning rate to 0.0005.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 67.1710 - vae_r_loss: 64.2954 - vae_kl_loss: 2.8756\n",
            "Epoch 7/30\n",
            "\n",
            "Epoch 00007: LearningRateScheduler setting learning rate to 0.0005.\n",
            "60000/60000 [==============================] - 6s 106us/step - loss: 67.0499 - vae_r_loss: 64.1767 - vae_kl_loss: 2.8732\n",
            "Epoch 8/30\n",
            "\n",
            "Epoch 00008: LearningRateScheduler setting learning rate to 0.0005.\n",
            "60000/60000 [==============================] - 6s 106us/step - loss: 66.9841 - vae_r_loss: 64.0982 - vae_kl_loss: 2.8859\n",
            "Epoch 9/30\n",
            "\n",
            "Epoch 00009: LearningRateScheduler setting learning rate to 0.0005.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.9496 - vae_r_loss: 64.0505 - vae_kl_loss: 2.8991\n",
            "Epoch 10/30\n",
            "\n",
            "Epoch 00010: LearningRateScheduler setting learning rate to 0.0005.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.8580 - vae_r_loss: 63.9414 - vae_kl_loss: 2.9166\n",
            "Epoch 11/30\n",
            "\n",
            "Epoch 00011: LearningRateScheduler setting learning rate to 5.000000000000001e-05.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.8192 - vae_r_loss: 63.9122 - vae_kl_loss: 2.9070\n",
            "Epoch 12/30\n",
            "\n",
            "Epoch 00012: LearningRateScheduler setting learning rate to 5.000000000000001e-05.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.7985 - vae_r_loss: 63.8998 - vae_kl_loss: 2.8987\n",
            "Epoch 13/30\n",
            "\n",
            "Epoch 00013: LearningRateScheduler setting learning rate to 5.000000000000001e-05.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.8000 - vae_r_loss: 63.8877 - vae_kl_loss: 2.9123\n",
            "Epoch 14/30\n",
            "\n",
            "Epoch 00014: LearningRateScheduler setting learning rate to 5.000000000000001e-05.\n",
            "60000/60000 [==============================] - 6s 106us/step - loss: 66.7863 - vae_r_loss: 63.8725 - vae_kl_loss: 2.9139\n",
            "Epoch 15/30\n",
            "\n",
            "Epoch 00015: LearningRateScheduler setting learning rate to 5.000000000000001e-05.\n",
            "60000/60000 [==============================] - 6s 106us/step - loss: 66.7805 - vae_r_loss: 63.8606 - vae_kl_loss: 2.9199\n",
            "Epoch 16/30\n",
            "\n",
            "Epoch 00016: LearningRateScheduler setting learning rate to 5.000000000000001e-06.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.7642 - vae_r_loss: 63.8401 - vae_kl_loss: 2.9240\n",
            "Epoch 17/30\n",
            "\n",
            "Epoch 00017: LearningRateScheduler setting learning rate to 5.000000000000001e-06.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.7645 - vae_r_loss: 63.8498 - vae_kl_loss: 2.9147\n",
            "Epoch 18/30\n",
            "\n",
            "Epoch 00018: LearningRateScheduler setting learning rate to 5.000000000000001e-06.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.7488 - vae_r_loss: 63.8296 - vae_kl_loss: 2.9193\n",
            "Epoch 19/30\n",
            "\n",
            "Epoch 00019: LearningRateScheduler setting learning rate to 5.000000000000001e-06.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.7785 - vae_r_loss: 63.8611 - vae_kl_loss: 2.9174\n",
            "Epoch 20/30\n",
            "\n",
            "Epoch 00020: LearningRateScheduler setting learning rate to 5.000000000000001e-06.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.7750 - vae_r_loss: 63.8555 - vae_kl_loss: 2.9195\n",
            "Epoch 21/30\n",
            "\n",
            "Epoch 00021: LearningRateScheduler setting learning rate to 5.000000000000001e-07.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.7647 - vae_r_loss: 63.8493 - vae_kl_loss: 2.9154\n",
            "Epoch 22/30\n",
            "\n",
            "Epoch 00022: LearningRateScheduler setting learning rate to 5.000000000000001e-07.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.7589 - vae_r_loss: 63.8436 - vae_kl_loss: 2.9153\n",
            "Epoch 23/30\n",
            "\n",
            "Epoch 00023: LearningRateScheduler setting learning rate to 5.000000000000001e-07.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.7746 - vae_r_loss: 63.8587 - vae_kl_loss: 2.9160\n",
            "Epoch 24/30\n",
            "\n",
            "Epoch 00024: LearningRateScheduler setting learning rate to 5.000000000000001e-07.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.7662 - vae_r_loss: 63.8503 - vae_kl_loss: 2.9159\n",
            "Epoch 25/30\n",
            "\n",
            "Epoch 00025: LearningRateScheduler setting learning rate to 5.000000000000001e-07.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.7506 - vae_r_loss: 63.8357 - vae_kl_loss: 2.9149\n",
            "Epoch 26/30\n",
            "\n",
            "Epoch 00026: LearningRateScheduler setting learning rate to 5.000000000000001e-08.\n",
            "60000/60000 [==============================] - 6s 105us/step - loss: 66.7519 - vae_r_loss: 63.8338 - vae_kl_loss: 2.9182\n",
            "Epoch 27/30\n",
            "\n",
            "Epoch 00027: LearningRateScheduler setting learning rate to 5.000000000000001e-08.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.7990 - vae_r_loss: 63.8800 - vae_kl_loss: 2.9190\n",
            "Epoch 28/30\n",
            "\n",
            "Epoch 00028: LearningRateScheduler setting learning rate to 5.000000000000001e-08.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.7691 - vae_r_loss: 63.8507 - vae_kl_loss: 2.9184\n",
            "Epoch 29/30\n",
            "\n",
            "Epoch 00029: LearningRateScheduler setting learning rate to 5.000000000000001e-08.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.7882 - vae_r_loss: 63.8689 - vae_kl_loss: 2.9193\n",
            "Epoch 30/30\n",
            "\n",
            "Epoch 00030: LearningRateScheduler setting learning rate to 5.000000000000001e-08.\n",
            "60000/60000 [==============================] - 6s 104us/step - loss: 66.7614 - vae_r_loss: 63.8420 - vae_kl_loss: 2.9194\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6tj2q69CAgDF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#model.fit(x=train_images, y=train_images, batch_size=500, shuffle=True, epochs=20)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qjkYaD4_Akzh",
        "colab_type": "code",
        "outputId": "de542e52-3037-4eff-9914-cc39dd987f66",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "n_to_show = 10\n",
        "example_idx = np.random.choice(range(len(test_images)), n_to_show)\n",
        "example_images = test_images[example_idx]\n",
        "\n",
        "z_points = encoder.predict(example_images)\n",
        "reconst_images = decoder.predict(z_points)\n",
        "\n",
        "fig = plt.figure(figsize=(15, 3))\n",
        "fig.subplots_adjust(hspace=0.4, wspace=0.4)\n",
        "\n",
        "for i in range(n_to_show):\n",
        "    img = example_images[i].squeeze()\n",
        "    ax = fig.add_subplot(2, n_to_show, i+1)\n",
        "    ax.axis('off')\n",
        "    ax.text(0.5, -0.35, str(np.round(z_points[i],1)), fontsize=10, ha='center', transform=ax.transAxes)   \n",
        "    ax.imshow(img, cmap='gray_r')\n",
        "\n",
        "for i in range(n_to_show):\n",
        "    img = reconst_images[i].squeeze()\n",
        "    ax = fig.add_subplot(2, n_to_show, i+n_to_show+1)\n",
        "    ax.axis('off')\n",
        "    ax.imshow(img, cmap='gray_r')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1MAAACrCAYAAACDkcZ1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2dd5gsVZn/PwcFlSBBwkW8csmI5CSK\niCQBBTGAWREz6qprWNwVE6ZV2DUg6qqYFUXBFREFvIiy/ggSL5d8QbKAgEhSYv/+mP52nT5zqrq6\nZjrMzPfzPPN0T/fpqlNvnTrhTSe0Wi2MMcYYY4wxxvTHUqOugDHGGGOMMcbMRLyYMsYYY4wxxpgG\neDFljDHGGGOMMQ3wYsoYY4wxxhhjGuDFlDHGGGOMMcY0wIspY4wxxhhjjGnAY3t8P1fzpocBH99y\nHRyW7WCwXAeD5ToYLNfBYLkOBst1MFiug8OyjbBlyhhjjDHGGGMa4MWUMcYYY4wxxjTAiyljjDHG\nGGOMaYAXU8YYY4wxxhjTAC+mjDHGGGOMMaYBXkwZY4wxxhhjTAN6pUY3xhhjjBkLvvGNbwBwySWX\ndD770pe+1FXmiCOOAOAtb3kLAMsvv/yQameMmYvYMmWMMcYYY4wxDQitVuW+W96UazBYroPDsh0M\nlutgGLpczz77bACe//znA/Dxj38cgHe+853DrMask+vixYsB2GOPPQA4//zzAVhzzTWHWY1ZJ9cL\nL7wQgBe+8IUA3HrrrQA8/PDDpb/RvOaGG24AYK211ppqNWadXMcEy3UweK41OLxprzHGGGOMMcZM\nF0ONmZK26MEHHwTglltu6Xz3zW9+s/K3G220Uef9JptsAsBuu+0GwIIFCwA47LDDANh3332np8Iz\nlBNPPLHz/qyzzgLgc5/7HABrr702AEuWLCn9vTSpq622GgAf/vCHAdh///2nv7LGTIFHH30UgPvu\nu6/r85/85CcAnHrqqZ3Pjj32WAC23nprAH7/+98Dcy+e4r//+78BuOuuuwD45S9/CQzdMjXreO1r\nXwsU49opp5wCwIEHHjiyOs1kZIHSeH7TTTcBEEJvpfsHP/hBoBjDjDHjzT333AMU/ef8+fM73113\n3XUAHH300QD84x//6PrtokWLOu8333xzAN761rcCsOmmmw6oxt3YMmWMMcYYY4wxDRiYZUoaYyi0\nxt/+9rcBePe73z2t51JWn9/97neALVOPf/zjO+8//elPd3139dVXA4V276CDDup8d8ghhwCw6qqr\ndh1n2WWXHVxlZyHSSB966KGdz/70pz8BRXyKvltqKeszeqH+4ze/+Q0Ap59++qTv1LfUQVqtk08+\nGYCXvvSl01HNGYOe+Z/97GcA3HzzzaOszqzh/vvvH3UVZiyS3cc+9rHOZ9tvvz1Qr32+733v6/qN\nvSjMsNG4Is8gzQN6eV0BnHDCCcDcnrv+y7/8CwDf//73geJZhsLDqg5/+MMfAPj5z38OwG9/+1sA\nNt5442mpZxmeyRljjDHGGGNMA6bdMiVNsVaXAAcffHDt3y+99NJd/z/00EM9f6PjKxZgriJr4Gc+\n85lJ38m69JrXvAaA17/+9QDssMMOk8rqHuo4CxcuBGCllVbqlDnppJOmqdazB1lOXvnKVwLw97//\nvfOdLIHSvL7//e8HbPWr4owzzgDgAx/4AFBkoZsql112GQBPfOITp+V4M43Ycm3MOKD4hjj28dxz\nz82W3XLLLYEiRg3gBS94AQAbbLDBoKo4I7nooouA7n24jjvuOKB7fAJYccUVgW5L/bve9S4Atthi\ni4HWc6YSy3CvvfYCJo9TdWL8NC+7/fbbgcKTBeA73/kOUMT6vu1tb2te4TFEuRSuvfZaoJjH5qxR\nim9WHP/LXvYyAL7+9a93ysjKd9VVVwFw/fXXA7ZMGWOMMcYYY8xY4sWUMcYYY4wxxjRg2t38FFj/\nhS98oa/frbzyykBhvrvjjjsA+NSnPlX6myc84QlA4QY011F6SCXigGKzQiWeWGaZZUp/f/jhh3e9\nyuQs879M/qabv/3tb0Bhcr733nsB2GmnnTpldB9+/OMfA/DII48Ms4ozijPPPBOA9773vcBkd595\n8+Z13kuOf/3rX3se98UvfjEAr3rVq4Du+2OMGT7bbLMNAF/72tcA+OEPf9j57n//93+7yiqgfL/9\n9is9nrY7+OhHPwoUyQDmyvYHDzzwAFDITslm/vnPf04q+9SnPhUo3KDuvvtuoDuZj7aY0LYz6pPn\nKnLrk3zjea5cKoXcqZWqO4fSgP/lL38BiratkIEYJVWZbW5+X/7yl4Hi2a3immuuASZveRAnWkuT\nrg0LW6aMMcYYY4wxpgFD3bRXyDrynve8p/OZNozUinPnnXfueZzvfve7ADzlKU+Z7irOGqSRKwuC\njJN2/Md//AdQaPuluf/P//xPAJ75zGcOrJ4zEVlM3v72twOFRer5z38+UAT6QhFQKsvUV77yFaBI\nR28KZOmTNVXsuOOOQJFeHgo5Hn/88V1lZU2VNQrgi1/8IjB3E0+kKPBXm6EqYBccyG+mD3mZaHwB\n+J//+R9gssX+mGOO6ZSRVUnJD1Q2h9KnS7MvK4vSJKtPnu0sXrwYKJIgiV122aXzXmPOM57xDKBI\nyCPUT0JhmVLipA033BCAffbZZxprPXP40Y9+BMA73vGO0jLbbrstUFhc4hTfKTfccANQeHTFidtm\nM/HWSXrOxUYbbQTAJpts0vnssY+dWKpoXB9HbJkyxhhjjDHGmAaMxDIl7cZnP/vZSd/Jb/Kcc84p\n/b3See+5554DqN3MRSv6D33oQ53PFHMmn2dpmI444ggAPve5z3XKyiL1rGc9C4Bf/OIXQHdK9LlO\n7MusWD1p9L/61a8ChVbwcY97XOlx5PtrJqP2KwuVkGVUaf0BbrzxRgCWW245oIhbk9W7yl99rqJN\nuaXlu+uuu4DuzVFtmTLTxZVXXgnA9773vc5nF198MVCkP5ZHShyv0481SZrrFVZYASgsU9L0zxXL\nlMZzjT1vfvObAfj85z/fKfOYxzym6zfp9iiyrACss846QOGdIq+AuWaZOvbYY4FqTxLJWDG5aVxP\njvnz5wNwwAEHAHnL1BprrAHMrni12MspnetrHJKnT/x+3XXXzR5P7RwK67TGMG0/UydF/VSwZcoY\nY4wxxhhjGjDtlimtsGMt1J133tlVRn6icczUJz7xidrnWLBgAeDYhxRlN4zlKsuUXv/4xz8ChW91\nfG+UAend7343YItUzGmnnQYUGXUAHnzwQaDYgFcbT/7jH/8A4JOf/GSn7JFHHtl1vNtuu21wlZ3h\nPO1pTwPgwgsv7Ppc1tQYZUw6+uijAXj5y18+4NrNfDbddFOgiDVNN+80ZjqRtl39JRQbm2qzUsWd\n5jaRr8Pqq68OFPEXW221VbPKznA0TslCF2/WWxdZ+aDwUhHXXXfdFGo3c/npT38KdFtLoDubn+L+\nl1qqfxuFslnm0Nyh6bMxTmhMT+P0Yu65556u15iysUoeV+l7gIMPPhiAN73pTUAxv4Bizjwd2DJl\njDHGGGOMMQ3wYsoYY4wxxhhjGjDtbn4yC//sZz/rfLbrrrt2lZGpLk7BqaCzPfbYo+c55Epo8qyy\nyiqd9zJ56vX000/vKvuc5zyn817uanNlg8M6KDhfQaX3339/5zuZ9V/ykpcA8IY3vAGAX/3qV0D1\nRrJxkhDTjTYyLEPulAAf/OAHgcL11xgzHsil55e//OWk7xQkHocDTAdKo57WIXZPW3vttaf1nKNG\nSXigcKWsSn6UItc1uafFm8/K5d/kibfeaOLe93//938AnHzyyV2fy+01PcdMR264ORc+ob4hdsF7\n6UtfCsC8efO6ysr9UhtPw+TNvpUYTK+ve93rOt8p4U2Te5diy5QxxhhjjDHGNGBgqdHj1Ida/R10\n0EGl5bXC1GsVSq1u8sQpIGV5WnrppQF46KGHusrKmgK2SOVQWl+l2o3RhpDagDKVbQ6lmt1ss82m\nq4qzgnibBGnrUhRIqo16jTHji/pObQodJ4watMVDG1JfccUVAPzlL3/pfDfbLFNKJAOwzDLLAIWF\nSte95pprlv5+3333BQqPgDg5Uro9RTxfMM1RkipZXLQtzcYbbwwUG/7ONuRV9o1vfKPz2TbbbAPA\nhz/8YaDYziRN4Z/j2c9+NtC9CbA2Cf/IRz4CFBuByyMutoZrrqz6aJ7cBFumjDHGGGOMMaYBA7NM\nKWUxFOmKzzvvPKDYqO/cc89tdGz5RyrFr+kmtqJoE9Myq4k3j61m++23BwrN3p///OfOd4sWLQIK\njWsq49hCuNtuuwFw3HHHAf35tM9GpAXVxttK1Q/dWqaYF73oRYOv2BxC2nu9mqmx4447AoVFZq6j\njXnVD/7Xf/1X57udd955oOfWOdPX2c6rX/1qoEiJri1nYktHGh+i8ev3v/996XF33313AD796U9P\nX2XnGKeeemrn/WGHHQYUcdWyLv785z8ffsWGyEYbbQRMf4r9uE1rs2TFSKnf0VxYMe0A3/3udwHY\naaedAHjjG9/YvA6Nf2mMMcYYY4wxc5iBWaZilJVDm4/dd999AOy9996dMmeccUbt4yn2SvFVcUY6\nU2zQC4XmQ764r3zlK4Fig15pSADe/OY3A90+2GaCX/ziF8DkjWShiH9KN4p873vf23l/+OGHD7B2\nMwdZnRT/JOtejKzairv83e9+BxQZKZ/3vOcNuppzgrmmtR80F1xwwairMFYoplQoM+ooiDMH/+hH\nPxpZPQbNK17xCqCI49VmsJpzARxyyCEArLvuugAcddRRAKy44ooA/OAHP5h0XP1mOrKezRWU+ffQ\nQw8FCisIFLFoa621FgCnnHIKUFhuzPSx7LLLAsVzv+WWW3a+k6eR1hK2TBljjDHGGGPMkBmKZSpF\n8RJxnEQ/KNvM/vvvDxRxKPJ7nKtceumlQLcWTlpnaUW23nproPDfjTOnaW+w97znPYOv7AxD8Xm5\nOL3XvOY1Xf9Lfp/5zGcGX7EZxtVXXw1M3gvi6U9/eue9LFHan0b/n3nmmcOo4qxHmb4efvjhEddk\ndjF//nygsF7/+te/BuDAAw8cWZ1GiTJk7bXXXgB8/OMf73y3xhprAJP7zkFxww03DOU8o2aHHXYA\n4Atf+AJQeAB8//vf75TRe3lSKONf1f5+qdeFmWDJkiWd93r+zz77bKCQfc6bZZdddgGKuZo8h8zg\nUGzgD3/4w85nygZ41VVXAXD77bcDsOqqq/Z9fFumjDHGGGOMMaYBXkwZY4wxxhhjTANG4uanDcvS\nDeFyyPx5+eWXT/pOyRVe8pKXAEVaSZnu5hqSq9x4oEg9ve222wJFAKk2i4vd/C666KKh1HO2cPzx\nxwPFpnBCQcCPfexIHq+xI3YnS2WlANz3ve99nc+U2tQMBm1eqABpoVTWMPjU1bMRbaMg91SNT3MV\njSe5NMhyfRyUm5+2UVDioLnGW9/6VqAY/5V4Coqtac4///yex1Gw/nLLLTfdVZxRaLNihZQIhZoA\n7LfffkDR5jS/lUvrO97xjk7ZD3zgA8Dc3SLlT3/6U+e9Ep5VbSw9nSxYsKDzXvNhbRF05513Anbz\nM8YYY4wxxpihEXps2DiQ3RyVeEIppXPMmzcPgJNOOgnoTrNaliBh5ZVXBro3VlW6zz4ZdK7ggchV\nmyOfcMIJnc8kt+222y77mzgtsgIopVGUPKeRYeRgHtoOpNJWfec73wFggw02AIpECausssqwqgJj\n3Ga/8pWvdN7H2jkogvQVpB6z7777AnDiiSd2V2S4m8yOrVybcs455wCF9emBBx4AikQf8XcDZNbJ\nVcHo6gee+tSnAt0W/5VWWmnQ1Rg7uWqLAwXmx2hbEz3jyy+//FTq1iFN+/+sZz2r813sjdHPIaeh\nWlUMtL3qGYdyi+kRRxwBFBv+QtEHD9DCNyPk+tBDDwGw6667AvDHP/6x52/SxCtlc7ABMZZzrbe/\n/e0AHH300Z3PlI48nicMEm0XAEWSEHHFFVcAsOGGG1YdIitbW6aMMcYYY4wxpgFjG9Tx1a9+FShS\nckrbB4W1RTErQj6qjzzyyDCqODYoJbp89VdYYYXOd/1oQ6SxUhrZAVimZjxx7J5SycvvWZq9IVuk\nxhY9hwsXLpz03frrrw/ANttsU/r7NE5Smn4zNRQrJW212q82NzTN0Ob0yyyzDADXX389UFgCYW5u\nOC1LxwEHHND5TGOMxnLFRsdeJ4o9VUyFUkzHm24KeQNIu51aps4777xOWVnB9tlnn+YXNcOIY3Mk\nz5R43mC6WXrppbtecyhG+ve//z0Az3jGMwBvdAxF6nFtJi2LNHRbQgeJtmWJt2gQirHUvKQJvsvG\nGGOMMcYY04CxtUy97W1vA2DttdcGujeNk4YrtUyJeKX7sY99bEA1HB+UieSf//wnAE960pN6/kZW\nlZi9994bgM0333waaze7+PznP995f++99wKFpk/+5WYCWUDi51RZoQ477DAgn7nvm9/8JgA33nhj\n1+cf/ehHB1LPuYq09tLGDdmnf9ahzJTKJnvaaacBcO6553bKzEXLlNrVT3/6085nysB70003AcXG\nsYccckinjOJU7r77bqDYeF5a5Fi7/a53vQuAO+64I1uHo446qvNeMVzG1GXRokVAYWHJoVheZVW2\nRapA1uBHH30UgK233rrz3bCyHitWKt6cWpkW5Qk3lXvmu22MMcYYY4wxDRiJZWqjjTYCCksIFFm9\nxK233grA7rvvDnRbS6TNKkNZ1uYqL3vZy3qW0Uo8RntRmcnI6nfKKadM+u7QQw8ddnVmLPLLT/eC\nkzUKCi2zZK49UrQ3jZkayj5pzDCJLZ/SVCur109+8hMAbr/99k6ZtF+V5l9xF3qtQln8tAcQ1PPc\nMCbmE5/4BFA991SM8Cc/+UmgyPxnCu+pUfCFL3wBgK9//euTvtPegMoePhVsmTLGGGOMMcaYBngx\nZYwxxhhjjDENGImbn9JLfvGLX+x8dtlllwFw7bXXdpW98847ATj99NNrH3+6Nv6bqcSuEuKee+4B\nio19zzjjDKA7iDfdUNUUyIR/3XXXTfruxS9+8bCrMyPQc/60pz2t85mec5nX9azGySbk3id22203\nAB7zmMcMrrJzCAVIiyc/+ckjqomZq2yxxRZAkSzqne98JwAnnXRSp8z3vvc9oNj0WIknchvIajPk\nD3/4w0Cxcf373/9+wK59ZnhojDMFu+yyC1C48x955JGd75SUZ88995yWc2kLhX//938H4De/+U3X\n99pMGcqT2DXBliljjDHGGGOMacBIU6PHG/FqNa8Nz+LVY13e8pa3AHNvs1ltFinN/THHHNP5Tqmo\ntTqX5W/11VcHiuA8gBVXXHHgdZ2pxKl1U5Qu9dvf/jYAz33uc4HC+jJXefzjHw8USWSgeM7j9KQp\n6hekXY77CTP9fOhDHxp1FcwcZ8MNN+x6he4NfPtlKr81BTfffDMADz74IFBsSD3X8Bg0NbQVwmtf\n+1oAvvGNb3S+01ZHShynxHSbbrppp4y2nUjRZt2HH35457OzzjoLKLZbENq65oc//GHns+lMy27L\nlDHGGGOMMcY0ICjdaAmVXw4Cbeolbb9WkfFKVlpt+VnLEvW+970PmBYLS5jqAXowELlqQ1mlmQU4\n55xzgMIS9W//9m8AHHTQQcDQrXiDlisMSLaLFy8Gqjc0VruTZkRbAAyJsW2zDzzwQOe9rHcHH3ww\nAK94xSsA2GOPPTplXvWqVwGFZWvEjK1cZziW62CwXAfDrJer4s0+9alPTfpOcdirrLLKdJ92RshV\ncbyK/Tn77LNLy8qrKo7/GwFjOde69957ge446jheOkYb6kIxt3rNa14DwA9+8AMArr/+emBynHWM\nYi7XXXddYFpyKmRla8uUMcYYY4wxxjRg7CxTY8KM0JbMQMZSW1IHWUyVDRHguOOOA4rYKG3WF8cI\nDRG32cFguQ4Gy3UwWK6DYdbL9dOf/jSQ34RenkFvfOMbp/u0M0quixYtAuAjH/kIUMT1QpEZWfHp\nI/asmLFzrRmALVPGGGOMMcYYM114MWWMMcYYY4wxDbCbX54ZZXqeQdj0PDjcZgeD5ToYLNfBYLkO\nhlkv1/vvvx8oEgABXHDBBQAsXLgQgPXXX3+6Tzvr5ToiPNcaHHbzM8YYY4wxxpjpwpapPNaWDAZr\nSwaH2+xgsFwHg+U6GCzXwWC5DgbLdTB4rjU4bJkyxhhjjDHGmOmil2XKGGOMMcYYY0wGW6aMMcYY\nY4wxpgFeTBljjDHGGGNMA7yYMsYYY4wxxpgGeDFljDHGGGOMMQ3wYsoYY4wxxhhjGuDFlDHGGGOM\nMcY0wIspY4wxxhhjjGmAF1PGGGOMMcYY0wAvpowxxhhjjDGmAV5MGWOMMcYYY0wDvJgyxhhjjDHG\nmAZ4MWWMMcYYY4wxDfBiyhhjjDHGGGMa0GgxFUJYEEL4RwjhwpLvtwkhXBxCWBJC+FIIIWTKrBhC\n+GUI4aIQwiUhhINqnDe0j7ckhLAohLB1SbnTQwhXhBAubP+t3v78X0MI14cQvtzvNY+CnJxDCN8K\nIdwWQlhc8bsPRNe+OITwSAhhlUy5Ovfp1W1ZXxxC+H8hhC3anz+hffwHQwirTtc1D4MSue7VbjNL\nQggf7PH7l4YQWiGEbUu+/00I4a4QwokVx3hOCOH8EMLDIYT9o8/Xa8v13ibXNmxq9AV1n9lXttvY\norb8KttUWbvMlDu63ccsCiH8LISwfPvzGdUX5GjaPyTHeFII4XchhHurZBFCODyEcEsI4f3TUfdx\nYKryCyFslz6/yfc9+5QQwuejvvrKEMJd7c9nWz9QZ6zZOIRwZgjhgbrtrI/2u0X72BeHiXnHE9uf\n7xRCuLTu8zJspkOuUdnK9pqUrdtvfyqEcEPaTmdC/1pDtrXaYwjhjOgZvjmE8L81zn1gCOGq9t+B\nJWW2DCGc1T7uuSGE7dufv7x9X0rnF6NkmvqC54YQ/h7J9SM1zrtHCOG89rHPCyHsWlLugDCx5ng0\nRHO4KfcFrVar7z9gAbC44vtzgB2AAPwa2DtT5j+Az7bfrwbcCSzT47zPbx8vtI9/dkm504FtS757\nPfDlJtc97L+cnIHnAFtXyT8pvy9w2hTu07OAldvv905lDlwLrDpqWU1FrsBjgKuBdYFlgIuATUp+\nuwLwB+Csija2W1vuJ/aow+bA94D9M9/fO2o5NZFl5vuezyzwWOA2tSPgc8DHepy3sl1G5Z4Yvf9v\n4IPR/zOmL6gr+wb9w3LAs4G39ZIF8DHg/aO+7nGQX7vPOA04qeT5rd2nRL/5F+BbyWezpR+oM9as\nDmwHfKpuO6vbfoE/ATu3378B+ETdus90uUbtsbS9ZsrXnWvtAKyZa6fj3r/WkG2T9ngc8LoeZVYB\nrmm/rtx+v3Km3Cm6n+37cXr03XOpmF+MuVzr9AV9Xx+wFfDk9vtNgZtKyj0N2IjMOmEqfcG0u/mF\nENZkYgJzVmuidt8DXpQp2gJWaK9Kl2diMfVwj8PvB3yvNcFZwErt880ZWq3WH5iQVV1eCRyTflj3\nPrVarf/XarX+1v73LOAp/dd67NkeWNJqta5ptVoPAj9moq3l+ATwWeCfZQdrtVoLgXuqTthqta5t\ntVqLgEebVXnGUOeZDe2/5dr9wROBm6sOWrddtlqtu2FC0wo8gYl+Z9bSb//QarXua7Va/0dFe55L\n9CG/f2Fi4nRbyff99Cki21fPdPoYa25rtVp/Ah6qe+w+2u+GTCjBAE4FXlr3HONKH3Mt6N1eU2rN\ntdrn/kuD6o89/bbHtrVzV6CXZWpP4NRWq3Vneww7FdgrVwUmxkKAFekxJs4E+myzfdFqtS5otVqS\n0SXAE0IIj8uUu6zVal0xHeeMGUTM1FrAjdH/N7Y/S/kyEyvEm4GLgXe3Wq1eE8u1gBtqHBvg223z\n4IerTN+zmRDCskw8pMdlvq57n2LeyIQmYbZRq121XR3mt1qtXw2rYrOAnrJttVoPAQcz0Q/cDGwC\nHN3HOSrbZQjh28AtwMbAkX0c15hJhBDWAl4MfLWiWD9jFSGEtYF1mLAezDaajDXTzSUUi9kDgPlD\nPv8gqCXXmu01d+za7dcAE4uChVLgVVBXtu8BDg8h3AAcAfz7tNRytPTTFzwzTLjo/zqE8PQ+z/NS\n4PxWq/VAk0o2YZQJKPYELgSeDGwJfFl+zNPAq1ut1mbATu2/107TcWca+wJ/bLVa/ViysoQQdmFi\n0nrIlGs1AwkhLMWEm9j7Rl2X2UYIYWkmFlNbMdEfLKLmwFGnXbZarYPax70MePlU62vmPF8ADqmh\n/OuHVwA/a7Vaj0zjMU3BG4C3hxDOY8JV+8ER12eYDKK9mslMt2X5YOBfW63WfOBf6U/BONM5H1i7\n1WptwYQCtGccmmgvvD4LvHVAdcsyiMXUTXS73Dyl/VnKQcDxbTPyEuDPTGiOO4QQ3hEFoD25fZxY\no5Q9dqvVuqn9eg/wIyZcLuYir6D84a57nwghbA58E9iv1WrdMa01HA/qtKsVmPDDPT2EcC0TPr8n\nhJIkFHOVhs/slgCtVuvqtun/WCZionqdq3a7bE9Sf8wscO8xI2db4MftfmB/4CshhNRVpdZYFVHV\nV890ao81vQghvDjqX2r3va1W6/JWq/W8Vqu1DRNyvrrJ+ceMunLt2V6bzrVmExkZ9PPbVZmYZ9bx\nWqkr2wOB49vvf8rsmMfWarOtVuvuVqt1b/v9ScDSIUlKlesLQghPAX7ORNzaUJ/xaV9Mtf1n7w4h\n7NB2r3sd8ItM0euZCNQnhLAGEwFh1yTHOqrVam3Z/rsZOAF4XZhgB+Dvqb9uCOGxEnpb470PMJaZ\negZJCGFFYGfysq99n0IIT2XigX5tq9W6coBVHiV/AjYIIawTQliGiYnNCXGBVqv191artWqr1VrQ\narUWMBGn88JWq3Xu8Ks7vjR5ZpnoTDcJIazW/n8PJqxIpdRpl+1zrq/3wAuByxtfnDFAq9VaJ+oH\nfga8vdVqpZrTnn2KCCFszEQg+pkDrPbI6GNOUOdYP4/6l9p9bygy+i4FHAp8rcn5x4m6cq3TXhv2\n27OKjAz6YX8mEibUiT09GZxI6KcAACAASURBVHheCGHlEMLKwPPan6XczMQcDiZisa7qs05jRx/z\nznkKzwkTWQyXAu5IjtXVF4QQVmJiMfvBVqv1x4FfTMKg3PzezoTGeAkTGqBfA4QQ3hZCeFu7zCeA\nZ4UQLgYWMmGGvr3HcU9iYsG1BPhG+zy0j600jI8DTg4hLGLCjfCmdtlZQQjhGCYG3Y1CCDeGEN7Y\n/jyWLUz4SJ/SarXuqzhcnfv0EeBJTGizLgwhzLrFQ6vVehh4JxMd2mXAsa1W6xKAEMJhIYQX9nO8\nEMIZTGiSdmvfoz3TY4WJNLU3MuG//z8hhEum74rGip7PbHvg+jjwh/ZzuyXw6R7HLW2XIYST2prF\nAHy33cdczETWqcOm68LGkT76h/g31zLhwvr69m82aX/+zX60/7OBJvLL0Wef8grgx22r7Gyl51jT\nnkDdCLwXOLQt/56u/zXb7ytDCFcyoUy5Gfj2tF7d6KgzhjehzlyLEMLn2vds2bbsPzaFc44VVe0x\nGmNEbctyO+ziE0woXP4EHKZQjKTNvhn4rxDCRUyMh2+ZjusaA+q02f2Bxe1r/xLwihr94zuB9YGP\nhMnbInXkGiYsWjcCzwR+FULILWT7JjTpv0MIC5hYhW86HZUYJiGE1zORDvGdo65LL2aCnNsD2bY1\nFsJjwwyR672tVmv5UdejFzNBlmXMpL4gx7Bl354o3dtqtY4YxvkGzUxou+4HBs84132c69aLce9f\nZ7hsn8tEuvZ9Rl2XlBku1wU0rHtTy9QjwIqhZFOucSWE8K9MBLb3yrYyLoytnEN7015gaWZeeu9x\nlut67XrdOuq61GRsZVnFDOwLcgxN9iGEw4HXAFWW7pnG2LZd9wPDIYSwE/BLYFyVgTNVrjOhf52p\nsn058BXgb73KjoiZKtcp9QWNLFPGGGOMMcYYM9cZZWp0Y4wxxhhjjJmxeDFljDHGGGOMMQ3wYsoY\nY4wxxhhjGuDFlDHGGGOMMcY0wIspY4wxxhhjjGmAF1PGGGOMMcYY0wAvpowxxhhjjDGmAV5MGWOM\nMcYYY0wDHlv1Zau9o29uY98QQvZ/lY2/r7MxcHq89LfxMdJzpSy1VLFG7GdTYpVdaqml8pWZJh55\n5JHSSpXJoR/iY6TH0zU++uijPc+dyq6qbrn7nn73mMc8ZqByBXj00Udr3/B+rqfqGejVHnNlpku2\nuo+Dlu0tt9wyrbt7N5FZ+nmuTFXZJn3BmmuuOVC5qr1W3eMymvaxvWSWq8d0y3XQ7TXtY8dVrmk/\nPO597NVXX117TjBVesk+Nycoa7dN66bfr7/++gOVazrXqqrvVOdaZcfJHaNXPxDPtVKq6jesudYw\n5wO9ftMPU22vg5YrwMKFC/uWbT/tsw5TnZc1Odfuu++ela0tU8YYY4wxxhjTgErLVEqd1XI/K+o6\nK0Zp7qaqfUl/O91atH7oZ8XcSzOUo45mKdJkltZLss9pn6q0Y2VlxoU6mouptI9IO1R63FRuVe27\nqsyw23EdS5KuO1e2H4tcr/PE75tY+upoVceJOtpqkbOOpNdb1Rek9NMX1KnvoJlKH9vvcXrJPpZZ\nL1lVWZ2qygybqdah7Prjz6ssJr3q0I91sIpxkLUo8waqour60+/6mWs1nQsOW579PL9NnvWYJu2z\nn+NXlR238Wo65n515g/p+XJlm4xLTdqpLVPGGGOMMcYY04C+LFP9xC2V/a7sN3XK1KUqHqjquONg\ntWoSf1BFKov0/+m6F+OiGdG9y7UBUXafm2p6ys41Hccoo0rzOgyq+oJB161Ke11lyS3zex8nDd90\n+3dXxSvUiZVoEiuQO17VZ8NgquetM26kFv0mmuk65+n3u0FSp83U+X2uTfbqR3IW6qloxKu028Om\n6tlpIteqY/cz3lfd4yZeSaP0rKgTM1b1+15lqtpir1jJKu+iOuceNdMh0zpW6pwXzFSe/ance1um\njDHGGGOMMaYBXkwZY4wxxhhjTAP6cvPLUSfwOXV/eOSRR7r+B3j44Yezv5cZLz6PAqXTYOill166\nZ/3GnV71rTIZ5+Sqz9L/JcNll122811ZQP6gEjUMklydy66vTpBulVtCmStlnQDcqSb3GDZN6lLl\nClJ1jWqr6Xdxm9Z79R8PPfQQAI997ETXFidVSPuNKtmPA2UJB6pkVuVeUdYP61Uyi9+nMso9V1Op\n3yiYjuQy8XWkckzHsic84Qmdsqlc0zber3vPOCb5Kat3ri2mMsvNCfSZ/s+59kjG6bMteY97gqQ6\nY2zZmBSTPuPpa+73+i733KbzL73m5lppfXP/j1rWVWPRVEJX4s9SmadtHYpxKn3+Je9YvmmfUZUo\naJzGryqq3Hklk1wfm4ZxSCZx+47Hsfg3ub6gLEygSZu1ZcoYY4wxxhhjGjBly5Qo03pCoVF64IEH\nALjvvvsAuOuuuzpl/va3vwHwj3/8o+t4j3vc4wBYYYUVOmVXWmmlrs9UJl3dx+/HKa1sP5RpkWCy\nXCXPO+64o1Pm9ttvB+Cee+4BCvkuv/zyACxYsKBTdv78+V3fSQOSW5mXpVmOGXdZ10mUUFUm1Zj+\n85//BIr7Ie0TFJomvS6zzDJAIeNYm1KnrY5Ktv1o7foJMk4tqFDINZWH2jDAnXfeCcAtt9wCFDKX\npnq11VbrlH3iE58IdFtjoV7q6lFSJ5i3rJ3Gcn3wwQeBoi9Q36D2+vjHP75TdvXVVweKPlbtNcdM\nTTWdUkfbrDaZa4OS51//+legeKbXXHPNTtm0j1V/kGv/dfqBcQjkr1s2Z1FW27v33nuB7jmBxjF9\nJtmrz4znBHrOn/SkJwGw3HLLdZUdp+e5X8osUqnlI/5MstIzLzlD0Xb1nfpM/TZ+1tWPqm/Qdzpn\nPG7NhLlWleW3ykJXxwNLSPZpX6F+AuDvf/87UNwDyVHy1lgFsOKKKwLFPVCfUbWtzbhRNY9V+1Mb\n1fgU9wV6rzlWasGL5aX3qdwkr6pkX1PxwrJlyhhjjDHGGGMaUMsyVcffuEpzl2pCr732WgCuuOKK\nTpnrrrsOKFbsQiv1pz71qZ3P9H7ttdcGCs3fKqusAhRaKZi8is/5m45T/EmK5JuTq2T1l7/8BSjk\nefnll3fK3HjjjUAh37vvvhso5Lrjjjt2yj7vec8DYKONNgKKlb0sf7EGvyymrWmq1Ommqj5lfrJ1\n0vLmLK73338/ALfeemvXa3yvpEVdeeWVgaKtxtpVkcp0nGJ8+ok9yMU7lqVCjuWa+pyn/vnSTgFc\nfPHFXa/S/qmPiPuNjTfeGIA11lgDKJ6BnHZ12DSJ6cjFQaTa1VhW0vRfdtllQCEz9cvrrLNOp+x2\n220HwLrrrttVl9QfPT5XlUZ6VJrTOnWp0pqmMT3yqlCfC0W/e9ZZZwFw9dVXd53vmc98Zuf9Hnvs\nAcB6660HFFbSqme77DV3LcOinzmBXmNLvSxRanuS2aJFizpl/vznPwOF14ranmS26qqrdso+5SlP\nAYo2rOdeFtZYcy3ryjjFoPRjYUzlG8tV2n0995ojaEwCuO2227q+i61WUIz7UFj69KrxS95BsZW/\nLLZnXOYE6fl7jVs50nieXF+RWls197rmmms6ZW+++eau36susljPmzevU1bzXI1bugexJ0GuXx4l\naWxT6rEWtzk939dffz1QzF/jflRl9DvNCTTXj/sCPfPqA+QNIOu1ZBwfp45V1anRjTHGGGOMMWYA\nVC5n62w6pu+kHZE2PvZ31CpcGjxpRGPLlDRU+r1WjNIex/6mqWZFfqeqb86XNJcBKGVY/uf9bBCW\nruRjy90NN9wAFJpmvcYremmkdA+kWZW1KdYsPfnJTwYmx0nk4npElRanTkzSoKkTX5KrZ9q+07go\nKLQlite58sorgcLyKs0UFJo9afpl/ctpRdNMdHXayyjbbFmMVJ1+oyo7WvrM6jmXfAHOPfdcoNBm\n6f7oNdYcSnslbZbqGWdbGzZlvvg5yjT+uc8kK7VRKPoFyez8888HCs+BONZS2mlp9NUXVGU7ymmi\n0zLD6guq5NrLOhpr+vVe7UkykrYZYPHixUBhVVG/rOc41oRuuOGGQKElTWMpc9dQZTkZlVzr9AOp\n5UQWfCjG+0svvRSAs88+GyjGMCjmELovek4lz9iarb5Wnhdq0xo3pZ2Gok1rDEzr3+s6B0E/ctV1\n6xmP5ar5geQrbf9VV13VKSNvFckqjZWKLVPS8svyF8dYp6RxVbl+IO3TR2mp6vXsVMWiqd76Hwp5\nSubqFyT72Jqt+5RacDQfkxUKij58k002AfJeVqPwqGjSZtWPxusDWezOO+88oFgXxHP+NEOq2lgu\nPl3ykoy1plAdNM+FQt5pTHDVNZVhy5QxxhhjjDHGNKDSMlW1ci+znEhDpIxGUGiSL7nkEqDQ4Mea\nUK2s07innOZOq1pZBKTlk+ZZ/ry5+o4DVdm5UplrRZ7uZQKFNkMaCl33Wmut1Skj7ai0WKn2OF6R\np+eqiiXIxcOME/1oTlNZw2SNtNp17Huudr1kyRKgaNfSTMV+wbJMyTIov2e12dg6UmW1GRV1MsrV\nKRvLOCaXUU+fSY6yBpx66qmdsgsXLgQKLVRq0Y41fLqX0uRKMx0/A1VWgEFSZcVL/6/KjpZaAaSh\nhkJ+qQVVmrtYUy/tnvqNsrrA5L5gHLTOoo6GsWpPnrI9uXJaa8ler2lGtfTYuTpU7fFTxbBje3L/\nl8VMSR6yFkHR9qSNlkUqtqTqeBrXZCVJMyFCIWtZB+SJoWc7jk1V35BapqriwIZNbk4geaZZkeNn\nPI2fluVP8WdQ3Ae1L8lT8ojba5pRUb/R/CyOT9fv03nJTKHKmp3OjSSj2FNI1tUTTzwRKO6Bxp1Y\nVmqXkpWOo3PGlm+dSxZVzXfjOMBRUGcOUBbzH68PZLmT54T6gLiv1LMu66f6BP0fe02lsew6l+Qf\nW17juDPIW9vq9gG2TBljjDHGGGNMA7yYMsYYY4wxxpgGNM6nWGZ6ljtULmFEbMKHIqgRJqcylglT\nJmkFTcbHTt12VJfYZUim5lGlPG5KWfru2Cwpc69MxkoLG5vpJSO5nin5h+5JnICi56ZkGVesXvUe\nJ8rc/NKgfShcKNIkE3FyD7mqqG3KtURm5Tg1uu6D7pncUeQCU8e9Z5QB0ilV6c5Tcim8U1ewOIA0\n3bhb8rzgggsAOPPMMztl9Z3cL9JA/jj4Pw1+ztV31HLN0avdwuQU3ql7KhQyTreLkFzi9p8mAdFr\nro/tVW8Yz01ly9xpc+mT02QS8WbQSiajviJ1fY/dyXql4+4nNXZcftjttZ+gc/WlGnOgSBgjF2k9\n87ELjlJCP/3pTwe65wvQ3bbl6qbPNCeQm1ocUqBQArn7VW09MSy51nHpTudaGkNi13O5TF100UVA\nMV7F7mga89WGJVedO3YbTOds2oZG9ybn7jtOc606bqnpdzm3/1T2kkucyv+YY44BCnc/9cFy5ZfM\noGjnacIQzXfjc+s7PUc5l+BxSORR9V3q6hsn47jpppuAye73CoWAYisJyVDrA8kpbqfpZvTqWyTr\n+FlIk9HkZFs33GJ8Wr0xxhhjjDHGzCCmbdPeVGua29xUGgsF48eapi222AIoAsa1UpQ2S6tyKFah\nqVZfFppYOy2NYp2ND0dFlXylwdS1xYF20qxJ85ELeNZnOo40gmnAY4zkl6ZGr0rFmdPq1klHP0zK\ntLi5BBuSodqdrKFxSk/9ThqUtGzcZlONfhrAm9s4tmzz3vizUbfduA5liQdyWh5di57hWFbSIqv9\nSXOlYPXYSi3tn9qs5Cita5yMRf1OGnAdy34c5FlGes/j9lC26WCcXENaOAXv5ixSQjJK+52qjTir\n2sE4tdeUVJ65Piy9/lhzLPnJKqDrV5uOra7yLEitgnXuac4qOCq51kl/nz7bcRIEJZyIPVigW3u/\n8847A7DBBhsAxTOeS4esBBNKS68+JPUsgMn9QC6V97CTqFSdp2xulY5RUFy35KoycQIOebAoTb+s\nTbIaxHKV1UvjnqwHuf4l3QA1Z6ka9fNfdf6qcUvPu2QjT5+TTz65U0ZbTWhupX52s802A+DZz352\np6zGfY1pejZ0ztiaLUuqErCkydni341irlXVF5QlodGYDoV3icqorWpNALDlllt2fSc0/sfJODQv\n1jOfpmOPLbk6nn6Trhfia+npuVX5rTHGGGOMMcaYLI037S0rm0OrbFlQtKKWhgSKVbe+S1e08Wap\nsdULCu2IzpPzR6+j+RmWNqrMqpcjte7kNJdaTUtWsXzkVy3NhyxTWrXHspKmTpaWdLWeu4Y6G7SO\nC73kHd93XbNec6m2peGU9k9lpZmKr19aqnnz5gFFe5f/elVq8LT+uc+GnYa2SUwKTNay59L76r3a\nozRK0mrFcYPyPdf9UczlxhtvDMD666/fKZvKXP3GOPn490N8zyVjtUH1hfE2EdqsUM+3yqbWUig0\ndpJVqrGrEw+VS++cq/sgqOqXyixoVRZgofEplpXaqbTXaqcas+JjqO3mUvqW1TdllP1A1diVWqQU\nqyDrkKxGUIxLkqe075tuummnjJ7dNAW0ZBdbVPU+9SDQPYjjtdSHqz9Q39Fko85hUBbbl86R4u90\nTbpGvUIhY/WVetalsU/TRUMxp9D9T61QMXU2Ih/2XKtqLlLmqRLPoyRjtStt86P5FBTPvzyuZIna\nc889gSK2EibHDqmfVn8gayzAVlttBUyOFxr1uFVn7pfG8GpuFMcwytKsdidvknh9oP4hPbfaYdyO\nNGbpGUj7o3iuoTmcXtPNe/thZs4ijDHGGGOMMWbE1IqZqtIspNaR1EoExQpRGhCtQOPjym9SGiut\naLVyj7OoSfskzUpZjA9Ux0qlZcYpc09qkcrVO83glW4AB4VF6te//jVQZPnRbxYsWNApK3lqlZ5m\nRqvKODcOcq1LWZuIr1dtVpYPyTrOfigZSFuljfYk21gm0jxJ2yIrQZUmpEp+o5JpP+et03aVfSv2\n+9d90O8VI6U+Iq6D+hlZ/LbffnsAdthhB6CQMxTWhHSzzlydh0VVtilRpw8T6gN0rbFGVvEOepXF\nQLKPrS2Sm/rsNJtfrn4qk3oOxGXGod2WybMqnja1/MVlpFnVGKUxK70XUGitU4v0VO/7sKgau9LP\n1A9KExzHOuq5V/8nrbvieKDQwKfxarElRqSxJnpV3xzHaKhPlwY8p+kfh816U9JMkOncCwp5ylqq\nthf3g2nMpOSZyw4oi55kpnNrfMzF7YxTexV1LOpl2SiheMZlXVUWv3iuJc+rbbfdFoB99tkHKOZV\nsSV18eLFQBEzpT5Y8WvbbLNNp6wsU7qHmj+PMsYvR1WfoDamMSeOlVQ/oT5RMoitUek9UbtLLV/x\nuXS81GodZxLUc5J6reS8sZzNzxhjjDHGGGMGgBdTxhhjjDHGGNOASje/KrNW6tqRBuznTM96lRlO\nbjtQpJrUZ2ma6dgsmAbZpoHUuWuoMoOOmxsaTK5Tzkydut4oqDEOttUGcgqYlNuDzPR6hSIwV59V\nmevr1K/uZmfTSZNz5lxt1FZl9pWM47YnE7M2703T8sbIRK+garlN5M6dtuNxcpcQTTcOTctKznE7\nlEvFNddcAxQpp9V2Y/cLuehssskmADznOc8BYP78+ZOO20/CiWHLvM7Gh1WoXaYBuXE/LHcHuVvF\ngbjQHXieuj2k/WdV0HGuTY/KfbLO5rJVlG1anHNR1ximsUvyjNugkn+oLcYp1ntdy7j0sWWkz4zc\nSTUexZvsqr1KHnK5izftzQWXQz7xgj5Lt0dRGdUFJrd/jXtNXHumi37aomSXm/foumN39LgsFIH4\nant6VTKFOIW9+ly5+8otKt3wGMZ7nOonsUvOTVltV678kmG8saza8DOe8Qyg6ENVViEWACeccAJQ\nuAZrHJO7a5yIRZ/pnlZtTzEupHNHPX+ax8fPbrohutpW7Bapua3ukdq5ZKvvYbIrqsqk82SAm2++\nGSjmC7qfTdq1LVPGGGOMMcYY04DGm/aKsgQUsZYz1aBolR+vENNN9qTdUxmtLqEILtVnCj7Ta6zt\nU33qWNnGkbRu8Yo5TTyh65cMoVh5K7g03fgvTp0sJHtpo+qkQR2HAMiYOqnwpcGo0l6lGpY4AF/B\npwomveCCC4BC5uuuu26nrALP9ZomQahK7lF1TcPWSFclGimzZMZ1VJtNtX+x5lSWqOOPPx4o5Kvn\nPdZeSxu4++67A0VK9JwVqszKkEsnPSq5Vln8qiyWOW1qejxpmZcsWQIU/YT6agX+QrfGNXfOnFzL\nyuauZZT0kmv8v+RatnEqFP2AAvdlWVW/GW9OL+2ynoO0L5opfWydOqSpiDUGQXG9GmMUbC7tdHwO\nXbfkqvEpTloj0jTnmj/E3gLqG1QvjZuxBXFUMs6dN20T6VwrZ33Xq7T7sVUwtSBLDhdeeCFQ9A9Q\nyDi19Pez/cwoU3hXbeHSa0uH2DKi9qPkBXrm4+QyasOyulxxxRVAYQk99dRTO2X1neQo65M2ql1v\nvfU6ZdNkNXW2cBgFVXMDPWPqA+L5vNqzrHPpvBOK/jZNua52nZvza1zTuKeycV+ge62+W311vH6x\nZcoYY4wxxhhjBsiUN+1NtWVaTcYrO63etQLV6lKaJihWllptS3OvFWmsCUlT8MoPUyvO2NqSpk3P\naffqxFUNiybnTq0m8f3Sal9aE5XR/7EGWnJUanTdtypf/ar6jkKOU9GAV1mH1NZibaj8py+++GKg\nsEipbJzaM91wr07a8zobj44TZfXOWU3UDvUax0+ec845QGGRkhZQ90TtE2CXXXYBYLPNNgMmb7sQ\nWxLLNjQdB2uJqHNf68QBqT+N+1hp6KQV1Llk6Yvba2o5qdLwlt33qj52WPTTP+XaTFrfdDuK+H26\nIa/aaSxXxQ6k6b776bfqaNcHRZ1+S/JTO5OmPt4GQtettqfXXKrtNEYqtRbGv9O9UPyqLAlxjIY0\n1OpXVM94c+BxkqtIn6s0rhcKy57kofEqtgTIIi1rgfpepY/PbVORzuFyfbtkX2cj6nGYa6XtNW1n\nseeUZKQ5kuodjymyuig2Su3s0ksvBbpj0XQOxVIrFfp2220HdPcZZTkBxtnbKpVtOr/PbTGjPkDz\n99jql27JkW4/E7dDWU/Vjm+55Zau16Z9QS+Z2jJljDHGGGOMMQ1onM1PpP6bsijJ7xEKDZ1WiloZ\nxto9aUBWW201YLL1KtZcy8dXfo7KiKL4iXiDOtUj9RMeB43IVEmz+uhatUkcFJsgLliwACjujzaY\ni7VIsqykmXtymyGnGbvGQdOUq0e/1jSRZkfLxY1JS5XGpaUbJ0KhbUktJ1U+8uMUZyLqaPqrNj9M\nNVXqC6RJhsIyJQ2qykqGO+64Y6fsXnvtBRSZk2I/937rW/bZMGlqhSyL+8ll8VKfmmb8i++Tyuqz\ndLPanBV3qtcwSPqx5lRlyVK/EGtW1aemlilpSONMchrHUo1/Tq5pW9SzM66xaKIsviyuo+SYyixn\nmRJqgyqj+BIo4lOPPfZYAM466yygmHPEbVTeGLov6oPGSYYx6ViRyi7WoquP1NiTi1HR86++UnLI\nxZzrvY4jTX4aw1Kn3vH7Ucm6yloueUhWcYyfLJyae6YW5vg7yVexUsr2G98D3R95VOyxxx5AMYeN\nnwORelaMQwx1L9I2q+uK5aZ2LGtczmtKpJ4Benbj+HTJ9LTTTgPgqKOOAgrPjBjVR/c6nT/EeNNe\nY4wxxhhjjBkAtbL5iTo+mlpxxpo7rTSl+dDqL9ZqqIy0edK2aJWvDF9Q+DdK6yRtn2KmYg1A6g+b\n890ftda0ijpayVTmccyYspNoHx5ZnfQaawikdUr9rNM4n5i0XqPM3BPXo5/Yg5wGTXKR1lmvsd9/\nmolLWg5pnWT9A5g3b15XmVTjH9e3Sts3KupkRyz7P24T6Xd6lhV3BpNjzyQzZerbf//9O2VlcVXf\nojabZvWqql/d74ZNmWWiyqqptliW3Q8Ka0jatmONdHquNPNhVVxRFaPab6qKqrFBpHv8xLJPNasa\nh/R5HHsh7aj6htQiE9+3NGZonMapfuYCal+5fZxSD5RYVukeZ/q9ZKcxDYox68QTTwQKOUtzHWe8\nS9tyanUdB6rmJ2qLGovi2BLJRl4TuczJqSVV/6tMbJGRBUb3J7Xi5Tw/0vY6yj61nxg/zUslh3gf\nPnlJ6Polu9halGZV1nHiGB0hq4v2pNpoo42A4l7Gx831OeNIlUeCrktzydhrJ2etgu6+MM3Om8YL\nbrXVVp2y6oeVXVl9QZplNT0HlMdV18GWKWOMMcYYY4xpgBdTxhhjjDHGGNOAvtz86gRz5wKfZeKT\naTM2OQuZ4WX+UyIKmV7jwLArr7wSKIICr7/+eqAwbedSrqcm/brXNQiqNjUVqatXlak8dWWM3apk\nVl1nnXWAwgSqMrGLgJD89JrKMFevXGKPUZr5+0kuULXZnK5L7TN2X02Dp9OUyJI5FO1Z5uk04Dy+\n33U2PxwHF4pe5ILq1aaUpvS8884D4He/+12njMzyujbJ/gUveAHQ7d6jtinXCsk33YYhPl5VqtNx\nlGs/2xCk/W/umU1dJXIurOnvU1eoXFtM3XtySSqG7cJax80n/b8fOcPkpBR6VX8Qu/mkyRiq0smX\nnXuUcq3Td6pOcqeRfOK2mP4u544mUvlqTIvb63777QfAb3/7W6DbZRXyqZPlEpf2yeNAVV3SZzze\nhkZjj5JQqe3F7TXdvkZzK93b2267rVM2TUqRtv+cm19VGxm1jHP11fXLvVFhI7Ec5EYq2SlMQn0n\nFG1Mv5cLe7p1AhQJE5QKXfODVM5Q7jY5LrKsSjQieaUu5bGrXbqtgfqAeK6VhlToVfPXOJX81772\nNQC+9KUvAUUbFnHfoHqmYS9NZGvLlDHGGGOMMcY0oPGmvemqNN0sMqcBEFoFxqtTWaS0GtX/Ol68\nSlWQubRZqYYltkxJDl2ZswAAD2tJREFUM6NzVQVOjsuKP6YqAUUq81TDBMVGaApwlPZF2n9Z86CQ\nq2S95pprdh0/p42uY2UbhZa/SVB8ThskbVMunb8+S7UkSvqhjXqhe6uA+FxV7bEf2Q6rzdbZfLWq\n35CWSBsYnnvuuUB3O0ytndLivfCFLwSKNOhQBAmnGxvmNKdlAa6jfN6bJEzJWaDT9ppalaFop9Ls\np8l+colCyiwoOUtIleZ02H3AVBLR5K6tSq7qL7Ulh9qk5BuPR3pfJteqcbOqnxpHZPFR+4r7QGn6\nNQ7JUh0HpseafJi8KWyctEap0FOLVKp5BlhvvfWAoh+Rtnycxv2YsrmW7n08j9IYpCRUGotyctXv\nlGRCiVPiuVZZQH6uf02tKrnnKf3doGVeJjsonmldv8ag6667Dig8nqCYa6aW+tg6KkuULFppwgPd\nC4CXvOQlAGy99dZdx809z+O6DY2oStGu13QriNgjSn2B+s3U0wcK+aiNpVvWqP+AcouUiO9Zmo49\n/q5fbJkyxhhjjDHGmAZMedNekWrw440K5QOp1X0uNXqqxUjTVUpbAHDRRRcBhVYrTa0YH7fO5nKj\nool2P/a/13vJV6/xtWpFL19qoQ2P9QqTN6SbP39+12usuU4tATOJMg16LLeyzWXjtLE33XQT0L2h\nNBRawTg1urTUqQYq9QWGyTKtsgKMA70saLGGaMmSJQCccsopQBEzlYuVkEb7rW99KwCbbropkN+A\nMvXNTjVYcb2q0oYPW651+tiy9pqzYqQW1Li/0Ge6/lTrHFvu0g1S6/jHjxNN5Cpy/UCaGjq2gKRb\nU6gNqmzc/tN0/+nnObmOY4xfnQ1D09ikuD9MrQKKf46tIkJjl+YCGqeOPPLITpnchpxQ3AuNYQBP\ne9rTgMnbVcwUyrwmYLJ1KBcTLe277pfatvrVuC+W1SC1guUsUzMNXZPkp/Tn2kD+mmuu6ZSV5UMy\nknzjWH7NpbS1h44r76DddtutU3bXXXcFJm/hkYvtT+PR68wbxwXVtcpKnW6LoL4g13YlS7V9WQG/\n9a1vdcpcdtll2bpINvE2CfIekhfWVOInZ95M2BhjjDHGGGPGgMab9pb572qVKSsUFL64WkVK2xGv\nvlP/aGmsLr/8cgBOPvnkznfSTGkVv8Yaa3S9xsdKN1scd8r85NOsZVCs5KVRkdYk1qymG/LJmnLp\npZcChXyhuC+SlY6Xs+6V1TNmFFqS6YqRSLX4ko20TlDIMt0YUhrPOLYntZqmmui43mWbdI4yzq9O\nrJRI40skJyg20rviiiuAws88Rm129913B+CAAw4Aimc5p+mXRkltVvKOrVB1ZDUqC3aTdptrM+l3\n8fWnWuvU77xqc+U6cqmjOR22JruJXHNxkWl/EB9X/WWqrVc71fgHhWU7jS2ukt04ylVUjQmyMski\nFfeHGrP0/Kt/jePLNGal7VT9ifqQHHr+tal3bBV4+tOfDkyO3R5lJto642lZO40tU+r/0ix88VxL\n/ausKroHssjEfbLkIIuAZFaVvXfcrCNlpB4pqYUkzuanOafmtSobe2ClWWglM41jb3rTmzplZRFR\nHXL3qRdxGxmHTH9V41EaMyULFUzOfqg+MtcXpB4TaufarLsK9UdxNuDNN98cmBwz1aQvsGXKGGOM\nMcYYYxrQl2Uqpmy1ltPuaTWv1b1WorG/qbJ76ffS5iljT2xB0apXmi7FUqy11lpAd+YarTRT7eEo\nff6baEtFLDNln5E/rzQpseY+jTlbvHgxUGSqieNP0kxA8lWvsuqlsptJMVTpnhlxm02tK9KkxrF7\naqM6jjSw0jppXzUob391tHlq71X7tAyLOvVUm1P7VMY+KLJuXXvttUDRN8TtXHukHHzwwUDRHtW+\nY59+yUS/TzV78T0t8/Mfh/ifOudN20psddJ7veq6c5pT9Q+pBTTeMyXdI6hK81lmQR0nudbJhJmL\nnVRbljz1KksKFOOa+uF0fIs1rCLtD9L6xt/1I/tBU0cDrrYjS7320Fkn2ndP/anaomQW968ao3QO\nPdvqB+K2nZ5bmRX33ntvoNjPBwrPAXmwpG08d73DJpcJs6wusWVKbU1tT/KN22AaG6W5leQdH09z\nAGUDVmxJbh+79LPcnmzjIE+R7tOVWirjPkNyTPvQXMY4jfvbb789AAceeCDQnd1X6L6k+yPm+oFx\nG7fqWNPVFtKYqdhKrVizNMN0HJ9+1VVXZY+vNhxb/1N0blmkd9xxx853yhQsK2JVzFQvGc+cma8x\nxhhjjDHGjBFeTBljjDHGGGNMA2q5+dUJhkzNerk0uyJNcQqTU0/L5CeTvlx9oNj8TGbUXXbZBShM\n+3FwW+rmV2eDsVEGUKZ1qUqTrffpBoi5dOcykyp4V+blOEWl3CVlBpVJP009D5PlmTP5jpvpucy9\nMpeIIk3xr8QTaRp0KMz6Sr8rc37sbtorqUS/afLT340ywUqa7EBtS6b3G2+8sVNW8kzN/3G6UgXs\nbrDBBkDhCqjjx2VTF0iVqePO208bGQVl/VIa3Jv7TK48cqeCybLXsy83LAXhxudK08hXbRxb9bz3\nk8Bk0FS5y6Sk32nsilNxy70vDUCX27TkC4U7ifrSMlfDXD2rrmUc5JrKSuOG+sfYRV3B5mkinjjo\nP03+ITdUuWUp4RQUbXr99dcHYKeddgKKMU1zAyjuyzht71G1ebMoays5l7B0PIjd0fReY5pSUaeb\npMfvt912W6DoI3JufmUu7LlrGNZcq44rmtqV2qmSlsRtUX2nZKdrjVP5y3VNbe9FL3oRABtvvHHX\neeL6aPyrSjlf1ibqtJFRkZ4/3TA+3qpH80zN/VOXSij6gnRbD/Up8ZxAZXSu1L1vww037JRVf5xL\nwtQvo+9FjDHGGGOMMWYG0jgBhUiDDLVSz2k5pc3IpUeUpkq/l1ZfwaLxBl7SlkjrpIB1rXZjbUFZ\nIPWoV+4xVVqTVPMep32XjKU1yWmI9ZmSI+g40pJIkweFRmWrrbYCCk2NtCfxql3vqwKoZwo5K0Ya\nhJpLAqH2q9fNNtsMKNpjbsPY9P9ckG6ZlnncZZtqRXObw8oaIplJM6QAZyg0eukGfblntyxBgu5X\nrs1WWQPEqDT9ddJf5+qrZzRtp7E1WX2q2qdQelgl8IGifyiz4lRZ/MZJrv1Yd3JaffW3qaUu1jJL\nrpKfnnv1z/KggHzfEB8/pxmtSvowTvJMr0HXKPnE1ya5ymKk5AdKzw2FVVXH1W+khY7HeSW3kFVB\nY5fKxONmneQfda53OmlikVIbjD0g0uQfuu64vSnAX9/J60d9cKzlX2+99YBCruq3U4sKVCf5EqNO\nOZ+rS2rplEdEnJwrLSvrSWx1luXj2c9+NlDILNeXlrXBqnrW/XxYTKUvyM211A7VF8QeLaklWzLV\n+Bb3x1oHyAKlflmWw7jfyFlY0/rVxZYpY4wxxhhjjGnAlFOjdw6U+B/HKz1ZlaSp04o93thX1pV0\nI7o0fWV8PK1ytdLUq7Qm8e9HvYqvoioNaqrlibVQ0iBJW6RYsjh1tN5L1tKoSFax37lW7jpe6kc6\nDr7l00mqJYvbrD5LtVaxBUXtWfdBvtGypsZWgV6+4jkf6XFqs3U0imofev7kC7311lt3yugzafHU\nDmO5SjOoPqBKdulmslXxBHViZcbRIlXWVnJxctLQSa5xCtptttkGKPoJldU9iS1TZZaTOnUfJ7lW\nUZYSPb52teU0dXHcF6p9S64ay9RXx/E6uh9lHhM5+o2nHAZ1LChpfxD3h2nadD3/udiedPsD/Ta+\nT2rL6bl1zvjcVXG+40haz7QNxnHPmhulc6Q45lyae43zap+aI8TzJ41x6iN03Nw9reP1M2pZ5+qW\nxpDKyhnLQc+t4oBlIYn7THn5aPwva5O5ejT1SBhX+ukL0rmoNtWNLYNpvK/QPYs/V/+gz9LXuN+Y\nzr5gds2OjTHGGGOMMWZIhB4r4r6XaVXxP2k8Qy4zncpIK50rm8Zg9aOFSj/P1T0MWAXw6KOP9pRr\nurKvE1eV87tOM62l2v54Rd9LW9rUvzeK4xi4aqVOm+2V1S9+n2ZFizNQSpMnzYo0ftLmxdbUtK2m\ndeg3ZiqT9XGgsr311ltbVXWLUVuTNineUE/XKdmkVlCYHP+TWp/iNpu2/TqZ2fqxpMybN2+gclV7\n7aerzT3nZdnAYk2/siUpA5LkKE1eHCvRKwtqHe1zlVwH3V7r9LEpdcYuEWc5lDeF5Ko+No0XgEJb\nXaePLRsDRinXq6++uksQuTaY/l9H667XeJPtdIyqip0sG5tUJud10I9c11tvvUGPXX33A50fZu6B\nZJXOp6Bopxq/1C8oK3Dc38oSpRg0WQJyMVO97kGuzoOea1XNB9JT67olq9hzSrJJs/nFnkJ6r7Gt\njkdPOrbVqWeuvWba8sDnWgsXLqzdWPux/KRzVpgcC6x+Iifbsjjq9Pv4u376gt122y0rW1umjDHG\nGGOMMaYBXkwZY4wxxhhjTAMqE1D0Y5qr4waSmuerXBrS1NRV7gRpatuqtL05eiUHGDS5a6sKpE9J\nXcdy7mq54Luy40wXoww2rXLZ6Qe55ch1L07nL1Nzmoa6agO4Oi6G/Xw3TqTPTy4tcZrKNOfeE7+P\nj5ejzvPRq74zjTryELHs5ZpT1t/l0sj303/WKTMsmU9l7KpzvFhWaZKKOkla6riViH7c5MaZKrck\nvcZB/7ntKKDatafMbaqfNtrv7wZNr/ZZlVRB5Nqr+mC57mmz9fiaNZalqajrJE6pYthzrX7GV11j\nPNbL3bGsLJT3mVXXOpXrr5MEZpDUCUGp+39MVaKpNBlNTuZlW6A0nRPW7WNtmTLGGGOMMcaYBlRa\npnKbwNalqUYwDRhtorFvqk0dlhaqyTXlVuvpZ3WsV1Xn6yddZxmj1u5VtdleFsgqDZ80/LHms9dx\nqlKhltWtTtmy3w2SqkDZsuvPBX+nz7WsUHGwaXqcKg1Tev1V8hjH1LJpe61KKlHH2lLVFzTRcIvp\n6guGRZ322qTt5ORblkymH01okwQkTX8/FfoZc8p+m76PqfIqia0AvY5XZvnKfVe3zoOkzrgl+hmn\nc2ORzpWmqU43qI5/V3afx32u1Y9cRe6a68xH+xnje11/Vf+Ss46Pgjrz2KrnsFfZfvqCfhjUc2/L\nlDHGGGOMMcY0oDI1ujHGGGOMMcaYPLZMGWOMMcYYY0wDvJgyxhhjjDHGmAZ4MWWMMcYYY4wxDfBi\nyhhjjDHGGGMa4MWUMcYYY4wxxjTAiyljjDHGGGOMacD/B1CbGfDdCN4qAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1080x216 with 20 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "36Tx7MA8p7Ct",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D16ORsukqRrt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1uzEC6FlqRv9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RmJz0vCkqR32",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uMDNBbx5qR0w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "input_dim = (28,28,1)\n",
        "n_conv_layers = 4  \n",
        "conv_layers_filters = [32,64,64, 64]\n",
        "conv_layers_kernels = [3,3,3,3]\n",
        "conv_layers_strides = [1,2,2,1]\n",
        "conv_transpose_layers_filters = [64,64,32,1]\n",
        "conv_transpose_layers_kernels = [3,3,3,3]\n",
        "conv_transpose_layers_strides = [1,2,2,1]\n",
        "z_dim = 2\n",
        "encoder = 0\n",
        "decoder = 0\n",
        "model = 0\n",
        "learning_rate = 0.005\n",
        "r_loss_factor = 1000\n",
        "img_x = train_images\n",
        "img_y = train_images\n",
        "epochs = 5\n",
        "\n",
        "encoder_input = Input(shape=(input_dim), name='Encoder_Input')\n",
        "x = encoder_input\n",
        "\n",
        "for i in range(n_conv_layers):\n",
        "\n",
        "  conv_layer = Conv2D(filters= conv_layers_filters[i],\n",
        "                      kernel_size= conv_layers_kernels[i],\n",
        "                      strides= conv_layers_strides[i],\n",
        "                      padding='same',\n",
        "                      name = 'Conv_Layer_'+str(i))\n",
        "  \n",
        "  x = conv_layer(x)\n",
        "  x = Dropout(0.25)(x)\n",
        "  x = LeakyReLU()(x)\n",
        "  x = BatchNormalization()(x)\n",
        "# int_shape in used because using Keras.shape() will output a Tensor but int_shape outputs a tuple\n",
        "\n",
        "shape_before_flattening = K.int_shape(x)[1:]\n",
        "x = Flatten()(x)\n",
        "\n",
        "mu = Dense(z_dim, name='mu')(x)\n",
        "log_var = Dense(z_dim, name='log_var')(x)\n",
        "\n",
        "#self.encoder_mu_log_var = Model(encoder_input, (self.mu, self.log_var))\n",
        "\n",
        "def sampling(args):\n",
        "  mu, log_var = args\n",
        "\n",
        "  epsilon = K.random_normal(K.shape(mu), mean=0.0, stddev=1.0)\n",
        "  return mu + K.exp(log_var/2)*epsilon\n",
        "\n",
        "encoder_output = Lambda(sampling, name='encoder_output') ([mu, log_var])\n",
        "encoder = Model(encoder_input, encoder_output)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2s2c8FYqYpV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "decoder_input = Input(shape=(z_dim,), name='Decoder_Input')\n",
        "\n",
        "y = Dense(np.prod(shape_before_flattening))(decoder_input)\n",
        "y = Reshape(shape_before_flattening)(y)\n",
        "\n",
        "for i in range(n_conv_layers):\n",
        "\n",
        "  conv_transpose_layers = Conv2DTranspose(filters = conv_transpose_layers_filters[i],\n",
        "                                          kernel_size = conv_transpose_layers_kernels[i],\n",
        "                                          strides = conv_transpose_layers_strides[i],\n",
        "                                          padding='same',\n",
        "                                          name = 'conv_transpose_layer_'+str(i))\n",
        "  \n",
        "  y = conv_transpose_layers(y)\n",
        "\n",
        "  if i<n_conv_layers-1:\n",
        "    y = LeakyReLU()(y)\n",
        "  else:\n",
        "    y = Activation(activation='sigmoid')(y)\n",
        "\n",
        "decoder_output = y\n",
        "decoder = Model(decoder_input, decoder_output)\n",
        "#decoder.compile(optimizer='Adam', loss='mean_squared_error')\n",
        "#print(decoder.summary())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q5pksHO2qbHe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_input = encoder_input\n",
        "model_output = decoder(encoder_output)\n",
        "model = Model(model_input, model_output)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qY2qoZW4qbyW",
        "colab_type": "code",
        "outputId": "49704e3f-660a-4d59-b7d6-6e2f487d45a7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 976
        }
      },
      "source": [
        "def vae_r_loss(y_true, y_pred):\n",
        "    r_loss = K.mean(K.square(y_true - y_pred), axis = [1,2,3])\n",
        "    return r_loss_factor * r_loss\n",
        "\n",
        "def vae_kl_loss(y_true, y_pred):\n",
        "    kl_loss =  -0.5 * K.sum(1 + log_var - K.square(mu) - K.exp(log_var), axis = 1)\n",
        "    return kl_loss\n",
        "\n",
        "def vae_loss(y_true, y_pred):\n",
        "    r_loss = vae_r_loss(y_true, y_pred)\n",
        "    kl_loss = vae_kl_loss(y_true, y_pred)\n",
        "    return  r_loss + kl_loss\n",
        "\n",
        "optimizer = Adam(lr=learning_rate)\n",
        "\n",
        "model.compile(optimizer=optimizer, loss = vae_loss,  metrics = [vae_r_loss, vae_kl_loss])\n",
        "print(model.summary())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_91\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "Encoder_Input (InputLayer)      (None, 28, 28, 1)    0                                            \n",
            "__________________________________________________________________________________________________\n",
            "Conv_Layer_0 (Conv2D)           (None, 28, 28, 32)   320         Encoder_Input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dropout_217 (Dropout)           (None, 28, 28, 32)   0           Conv_Layer_0[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_202 (LeakyReLU)     (None, 28, 28, 32)   0           dropout_217[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_217 (BatchN (None, 28, 28, 32)   128         leaky_re_lu_202[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "Conv_Layer_1 (Conv2D)           (None, 14, 14, 64)   18496       batch_normalization_217[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_218 (Dropout)           (None, 14, 14, 64)   0           Conv_Layer_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_203 (LeakyReLU)     (None, 14, 14, 64)   0           dropout_218[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_218 (BatchN (None, 14, 14, 64)   256         leaky_re_lu_203[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "Conv_Layer_2 (Conv2D)           (None, 7, 7, 64)     36928       batch_normalization_218[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_219 (Dropout)           (None, 7, 7, 64)     0           Conv_Layer_2[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_204 (LeakyReLU)     (None, 7, 7, 64)     0           dropout_219[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_219 (BatchN (None, 7, 7, 64)     256         leaky_re_lu_204[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "Conv_Layer_3 (Conv2D)           (None, 7, 7, 64)     36928       batch_normalization_219[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_220 (Dropout)           (None, 7, 7, 64)     0           Conv_Layer_3[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "leaky_re_lu_205 (LeakyReLU)     (None, 7, 7, 64)     0           dropout_220[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_220 (BatchN (None, 7, 7, 64)     256         leaky_re_lu_205[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "flatten_31 (Flatten)            (None, 3136)         0           batch_normalization_220[0][0]    \n",
            "__________________________________________________________________________________________________\n",
            "mu (Dense)                      (None, 2)            6274        flatten_31[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "log_var (Dense)                 (None, 2)            6274        flatten_31[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "encoder_output (Lambda)         (None, 2)            0           mu[0][0]                         \n",
            "                                                                 log_var[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "model_90 (Model)                (None, 28, 28, 1)    102017      encoder_output[0][0]             \n",
            "==================================================================================================\n",
            "Total params: 208,133\n",
            "Trainable params: 207,685\n",
            "Non-trainable params: 448\n",
            "__________________________________________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54JDvFJqsBwD",
        "colab_type": "code",
        "outputId": "ede8ad42-df81-41f1-fecf-4b294597954a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212
        }
      },
      "source": [
        "model.fit(img_x, img_y, batch_size=500, shuffle=True, epochs=epochs)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "60000/60000 [==============================] - 31s 516us/step - loss: 70.7351 - vae_r_loss: 66.5253 - vae_kl_loss: 4.2098\n",
            "Epoch 2/5\n",
            "60000/60000 [==============================] - 17s 283us/step - loss: 56.1323 - vae_r_loss: 52.4677 - vae_kl_loss: 3.6646\n",
            "Epoch 3/5\n",
            "60000/60000 [==============================] - 17s 283us/step - loss: 54.4114 - vae_r_loss: 50.5818 - vae_kl_loss: 3.8296\n",
            "Epoch 4/5\n",
            "60000/60000 [==============================] - 17s 282us/step - loss: 53.2697 - vae_r_loss: 49.3854 - vae_kl_loss: 3.8842\n",
            "Epoch 5/5\n",
            "60000/60000 [==============================] - 17s 283us/step - loss: 52.4571 - vae_r_loss: 48.5291 - vae_kl_loss: 3.9281\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f6b49fde1d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 143
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yDBEBr1UtAOC",
        "colab_type": "code",
        "outputId": "5427dc28-1293-4fea-a81e-cdb503fac98d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "n_to_show = 10\n",
        "example_idx = np.random.choice(range(len(test_images)), n_to_show)\n",
        "example_images = test_images[example_idx]\n",
        "\n",
        "z_points = encoder.predict(example_images)\n",
        "reconst_images = decoder.predict(z_points)\n",
        "\n",
        "fig = plt.figure(figsize=(15, 3))\n",
        "fig.subplots_adjust(hspace=0.4, wspace=0.4)\n",
        "\n",
        "for i in range(n_to_show):\n",
        "    img = example_images[i].squeeze()\n",
        "    ax = fig.add_subplot(2, n_to_show, i+1)\n",
        "    ax.axis('off')\n",
        "    ax.text(0.5, -0.35, str(np.round(z_points[i],1)), fontsize=10, ha='center', transform=ax.transAxes)   \n",
        "    ax.imshow(img, cmap='gray_r')\n",
        "\n",
        "for i in range(n_to_show):\n",
        "    img = reconst_images[i].squeeze()\n",
        "    ax = fig.add_subplot(2, n_to_show, i+n_to_show+1)\n",
        "    ax.axis('off')\n",
        "    ax.imshow(img, cmap='gray_r')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1MAAACrCAYAAACDkcZ1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2dd5Q0VZn/PxeQHCRIBpGc4+sLKiDg\nAsqqCAYwLq7ioiKrgis/80FRBBOsAVZdFcNiQASUFTxEAWEl5wwqQYKSEQnW74/ub9/bd6p7anq6\nZ6p6vp9z5nRP9+3q6qeeuuFJNxRFgTHGGGOMMcaYqbHAbJ+AMcYYY4wxxjQRL6aMMcYYY4wxZgC8\nmDLGGGOMMcaYAfBiyhhjjDHGGGMGwIspY4wxxhhjjBkAL6aMMcYYY4wxZgAWmuT9uVo3PYz4+Jbr\n6LBsR4PlOhos19FguY4Gy3U0WK6jwXIdHZZtgj1TxhhjjDHGGDMAXkwZY4wxxhhjzAB4MWWMMcYY\nY4wxA+DFlDHGGGOMMcYMgBdTxhhjjDHGGDMAXkwZY4wxxhhjzAB4MWWMMcYYY4wxA+DFlDHGGGOM\nMcYMwGSb9poG8bvf/Q6AL3/5y5O2/elPfzppm9e//vUA/OQnP5neiRljzBzj/PPPB+Dkk0/uvPaz\nn/0MgDvuuKOr7VprrQXAueee23ltzTXXHO0JzjAPPfRQ5/l1110HwA9+8AMAnnzySQC+853v9Pz8\nhhtuCMDOO+8MwKte9arOe7vtthsACy644BDPeG7yzDPPdJ5/7WtfA+Cwww4D4EMf+hAAhx566Myf\n2Bgzf/58AC655BIATjrppM57e+6556yck5ka9kwZY4wxxhhjzACEoij6vd/3zZng6quvBuCb3/xm\n57UTTjgBgPvvv7+r7Wc/+1mg22oSQhjkawf60BQYiVwH/K2T8sc//hGANdZYY7qHGrVcoQY6O0s0\nUmfL2G+//QBYdtllAXjFK14BROvzDDM2cq0ZYyfXc845B4BPfepTAFxwwQUA/OMf/+i0WW+99QDY\nd999Afjxj38MwA033ADAv/3bv3XaHnvssYOcRu3k+vWvfx2AY445pvPaTTfdNLwzAl796lcD8Itf\n/GKox02onVxHxX333dd5vtJKK3W9d8QRRwDw4Q9/eFhfN2fkWsZpp50GwCtf+UogzuHS6KKDDjpo\nkEOP5Vzr6aefBuD73/8+AGeddRYAP/zhDzttvvCFLwBw4IEHTvn48mwvtFDfoL1S2dozZYwxxhhj\njDEDULucqcsvvxyAr3zlK0D0QmlFmpJ7Yj760Y8CcMghh3Ree85znjOS8zTjzZ/+9CcgWoqV43D9\n9ddPaLvyyisDcNFFFwHw/Oc/fwbOcHxR7t8tt9wCwIUXXgjAi1/8YgCWXHLJ2TkxY9rcddddneca\noz7zmc8A8Pe//x2AN7/5zQC87nWv67SVBVosvfTSABx88MEAXHXVVSM645nnzDPPBOADH/gAUD6G\nC3mhV1hhhc5r++yzDwDPfe5zu9qeffbZAPzqV7/qvHbqqacCcMYZZwATvdgPPvhg57msz5K9MbOF\ncgdzNtpooxk+k3rzt7/9DYC9994biH3LYostBnR7kpTXp8epsNNOOwHR4zUV7JkyxhhjjDHGmAGo\njWdKK8HXvOY1ADz22GMALLzwwkCMiYYYd/7www8D8K1vfWvGzrPOfPGLXwSihwRg9dVXB2C77bbr\neq9KxT/lSA0hV6rWKKdBcbgAH/zgB4FuiyaU56Xde++9QMyDeN/73gfAm970puGf7BxEFY5UUerI\nI4+czdMZG+QBFC960Ytm6Uyag2T2tre9rfOaPKhvfetbATj++ONn/sRqxAMPPADAu971LqDcI7XJ\nJpsA8P73vx+Al770pQCsu+66kx5f0QKqBAjw6KOPArDNNtt0tVXfnFb+k5VbVRbXXnvtSb9zLqGK\nk2Ust9xyM3gm489c7yv6kVaVPProowE4/fTTgdhfyEutqACI8wR5oPUoL1YZ//Ef/wHA9ttvP/D5\n2jNljDHGGGOMMQPgxZQxxhhjjDHGDEBtwvze+MY3AjHZdPfddweiK3+RRRaZ8BmFEYgdd9wRmLsb\n9yk0rew1JfNWYa5t1vvUU08B8Pa3v33StmlBky222AKIRVMuvvhiAF7+8pcP+xQNcNxxxwEO85su\nuq/zPiEN51V52Te84Q0zd2INQHL585//3Hnt3e9+NxBDUaaCQljzYzUZFZFQWe3bb799QhuF3b3j\nHe+Y8vGXWGIJoLuMfC8U1p7LGWIqgWmhLVDSAl45Kqpipsc111wDRJmLFVdcEYibeM9l0nv2Ix/5\nSNd7eRGqj33sY53nG2ywARDXEipuM2/evJGcp7BnyhhjjDHGGGMGoDaeKVn85YmSl6mMr371qwB8\n5zvfAeB5z3seAJ/73OcAWGCBublGXHPNNYFY1rsKtkbHTSX7IS/U//7v/3Zek97ttddeAPzyl78E\n4gacZciivfjiiwMuz1uGPCaHH374LJ/JeJF7pHTvq3R3mniuNnqUt1qooA3AnXfe2fXY77ubjgon\npMU7tIHpVLbhUDl1bTKrSIxxsvzLQ58WRBIqgLT11lsD3eXjh8FRRx3V9T1mcs4991wgFuhIkTwX\nXXTRGT2n2UBejs9+9rMT3lP0yQtf+MJpfcdb3vIWAB555BEAiqK1/+0666wDxCJrcxEVlNN8PuXf\n//3fgf7e03ysminm5qrDGGOMMcYYY6ZJbTxTl156KRDjrMXjjz8OxA15Ab7xjW8A0bqvsohztbSv\nPFFVPFKyRqskbVmelYmoJL90Thv0QiybLo+UkPU6RZv+7rrrrkC0PMlCbQ9V5L3vfS8Axx57LAB/\n+ctfgFhi+Q9/+EOnrTdI7k/aJ+TWvNwT/aUvfanznp7Lq6DjqP/46U9/2mmbb52g7Rhmy0I4SlR2\nVxvRQty+owqS23777QfErRbe8573AOMVVaGxRZ7/++67r/OeNjZWiXl5RdNtTqayObe8otqOQnqr\n8srpsT796U8DsOGGG1Y+/lzg/vvv7/neP/3TPwHjpZ858shpm56ybVCm45G6+uqrO8/Vn+o75NU+\n9NBDBz7+uPDb3/4WKM+11PxptdVWm9FzqsL43hnGGGOMMcYYM0Jq45nKPVLi29/+NgDHHHNM57Xl\nl18eiJWA9t9//xGfXb3pl6vQC1mPTcy9S1loodatIY/UKqusMqGN4se32morAG666SagPHdCG3re\neuutXY+q5rPpppsO/gPGDPUFugbiySefBODUU0/tvHbggQfO3Ik1kIMPPrjzXNZQeVX65UbaY90f\nVYiqgqqFQqxKJc+MvCTppvTjgrxBN998M9DtqTzjjDOAeE8rp+6cc87ptDnooIOAWGVVfbA2SVaO\nD0RvdipriBW9TjrppM5r2vDTtJDn/2tf+9qE9zbffHOg2mbKTUfjt/KiUobhZVeuP8BDDz3U9Z42\nQ37lK1857e9pKrp3c9mk9POezjb2TBljjDHGGGPMAHgxZYwxxhhjjDEDUJswP6HEUZXiVEiPylBD\nLCc7VwtO5EgOKldcVopWKNRHJY/TpPS5Whpdcrv22ms7r+2www5ADCktY7HFFgNi8ZR+eINIM5Mo\nbCotFCE9V3EJ9QVpG6E+RI/6rEpYz7U+YhDyvhZiiJrC+z70oQ/N/InNMEsttRTQXR7/hz/8IRBL\nHatQRFqkQiWqFR6lkL17770XiAWAUpTQv+222wKxvLVD+3qjoge33XYbEMc1iAVBplIMpKlojqmi\nRmmho1Qvp4r0tN+8TJv0XnfddQBsvPHGA39fU3niiSeAbrnnKPz8iiuuAGLhnjqUkrdnyhhjjDHG\nGGMGIGizsB70fXO6PProo53n8+fPB2IJaSXoKtH317/+daftdDdMq8DEmpjDZaRyTS2AKjlbZn3u\nhUodX3DBBV3/D4FRyxUGkK0s9WUbwalM6k477TTlE7nqqqs6z7fffntgoofq5z//OQCvec1rpnz8\njEbrbBmrrroqEC3R6qtSK9SNN9446tNopFyl02kBikHQvZ9vuzDJuFGFRsq1Cvfccw8Qy/imHm8V\nolFhpals9FuRRslVeiW5nHfeeQMdZ5FFFgFiEYV//dd/HcLZddEouVZB3kAVPTj99NMB2GyzzTpt\n0jFsRNRGrldeeSUQN5JOkWdT84GpoHv9Xe96V882Ou4QPai1nGv1Q6Xpzz77bACOPPLIznu9+gV5\nE9M1wdFHHw3EDZBHQKls7ZkyxhhjjDHGmAGYlZwpeZ8Uhw+9Y1JVxncGvFFjQ5rPoOeyAP7ud78D\nohembKNfvfaSl7wEiOW75yKHH344MJhnKi3j2Stn6pRTTgFgzz337LxWtlmgiWgjb9Obsu0S1N/K\n26TtEfR6vzyoNddcE6i2MfhcRR4peaGVg6KNaQG+973vzfyJ1Rjp4mmnnQbAlltu2XlP+WVVUF7W\nbrvtNsSzG2+UryePlPj4xz8+G6dTG8q87irZr7FniSWWmPQ4irz68pe/3PO4/b5zrqFcvT322APo\njohSdMo3v/lNIEZaaY6l/gNiqf//+Z//AWI+2qixZ8oYY4wxxhhjBmBWPFMzEIdrMrTK16Os0Gl+\nVVp1CqIVWvkX47qR54orrtjzPW0MKTlVqWL29NNPA/D5z39+0rbf/e53ATjuuOM6r40gj6KRyFqX\nP/7jH/+YtXNqCvLo694dlLzyn45rItdccw0A++67LxA9UmuvvTYQK8qZiZx//vlA3LS4bMNO5U6K\nu+++e0KbBx54AIBddtkFiHo7lzdB7YU2pU3HHIjjvyp2zjXkfd95552B7g2khSJUlIu60UYbTWij\nPJ4f/OAHAFx//fVA/4iTSy65pOv4pjt3T89VW+HAAw8EotzS3GDp97HHHgvAEUccMfqTxZ4pY4wx\nxhhjjBmIWa3ml+aRqLKMYiM/85nPAHDSSScB0WoAcOaZZwKwyiqrjOrUalNhJifNWVBexLD225L3\nJfdQyZs1hNypWlaYkacjtchpLzOhGOm06p7yzrbYYouuttoX5aCDDqp8DqpeCQN7pmqrs4Oy3377\nAfD973+/dQLtvmqllVbqtLn11lsBWHzxxUd1GmMn1yrkfcEQ+wDRSLlqnDrxxBM7r7397W8HYjUq\neaRUgXaG90BphFw/8pGPADEHQnkOaZSA9qCSfIWqov3yl7/seXztFaS9qt7xjndM95QbIdde3H77\n7Z3n8+bNA+Cvf/0rEPeQ0t5f8hLOELWTq/aNfNnLXtZ5TflPGoP6eZmke6oNUKWtqvkNMb+nlnOt\nYaPxX95EiPNiebNUHXC55ZYb1te6mp8xxhhjjDHGDAsvpowxxhhjjDFmAGY1zK8KG2+8MQA33HBD\n5zUl9B566KGj+trauZ5FWgBB5SEvvPBCYHjhfjrORRdd1PW6vmca31Vr17PCySCWjVX4ybPPPjuh\nvTaKXHTRRbtef/LJJ4Hu0L1eqOR/KtsFF1xwKqctaquzg3LdddcBMSxFck3DJrQhokICR0Dt5KoQ\nvPT+VGGIYW2wnfcBP/7xj4FqBVgqUju59kNFJTTmpJugq6SvCs68733vG+ZXT5XayjXt41TC/Ikn\nngBimX4l60Pv8tPqixWWBrG/TrejAFhooVaNrbQk/Rvf+MZBTr+2cq3CZZdd1nm+zTbbdL2nYh2z\nVFymtnJNQ/0151TBgyrbl1QJCRzh+FXrudaw+M///E8ghgSnOMzPGGOMMcYYYxrArJRGnwoqCqCC\nFAA333zzbJ3OrKHNdlOL6Otf/3pgeB4pIet27plKi18M+zvrwFvf+tYJzz/xiU8A3fon5Hmq4oES\nKvOr8r5KAjYTUYnZAT11Y4s2gUzvTyXdplsdVEV9i46bHvuLX/wiMFSPVKO46667ANhrr72AuK1H\nmij+3//930B3ErSJqNDUe9/73s5r8kipmIwKTVXZDFX9QboZ8rbbbgvEflseBBUMOeywwzptB/RM\nNZrUKygUUbHrrrvO9Ok0grTYlGT0q1/9CohFKv7whz902qRzs8lQEbUdd9xx2uc5F9H9rUI2KfJy\nq+jMED1SfbFnyhhjjDHGGGMGoPaeqc0333zCa7IKPPXUUwAsvPDCM3pOs0HqFRLD3lxPm/L2srAM\nKx+jSXzyk58EYJlllgGiZQrKN/WDaPF7xSte0Xnt4x//OAB77rlnV9utttoKgAUWsF0j54QTTgCi\nFdt0k96P22233aTt1YfIE/Wzn/0MKL/flT8xrht19yPN29l9992BKDuNR8ohA9hwww2n/B3ySGvD\n3yOPPBKA5ZdfvtMmzfNpMiphfuWVV054T2XO8zyeqbLBBhsAsRy9jifPgTy3cw3lmR511FET3lt3\n3XUB2HTTTWf0nJqIPKby0Jd56jVevepVrwImlu5XjjXELQA87k+Nyy+/HIC9994bgMcff3xCG41Z\nU9maZhj4ShpjjDHGGGPMANTeM3XaaadNeM3W/BayLCt/aSqeI+VWlOVJ5MhKPY55UpOh+PyDDz4Y\n6K4ac/755wNw8cUXd31GG8gecMABnde0Ea8sWspFOfnkkwE477zzOm1f+tKXDu8HNBhtaKiKXE8/\n/fRsnk6tUZy49Ev3cplHuxfphrxz0Qt94403At2bdd5zzz1A9Ej95je/Abo3l+2FNvpMLdQ/+tGP\nuo6T51uOYx+77LLL9nxPuSOq3JnKXpsdp6/1QlECypHS9UlzWuYixx9/PFC+2bbmD2a45BX/9Kgc\nd4hVqucK8ihp7j5VHnjgASDOv3JPcxrxU6W/GAVzezVijDHGGGOMMQMysn2mVJ0HotU99YL04t57\n7wXgTW96U9dnVdkLoidghNbT2u19IE/SPvvsM+E9yWG11Vbr+j9lEEu18jD03UOQ95zY+6AfX/3q\nV4GJ8bzpHh/yWk2R2unssJBsjj76aMD7TPXrC3JSa2ieY6n7eZa8IbWT6w477ABEjzPESlCq2Ffm\nkVLc/oknngjAd7/73a73lbeSMn/+fCB6vF772tcC3RUB0xyLKVA7uSq3Oc0h1d4vVVBuTz9uv/12\noHw/QOiuEiiP4RSpnVyroHs8teRvueWWAFxwwQVAjKSYJRop136sssoqQJzLarxSPh/MSAXFWs21\nll56aSDeh8pFBTjkkEOAGImi/T7vu+++Tpuvf/3rQPRQCVVc1F5p0F1pdUR4nyljjDHGGGOMGRZe\nTBljjDHGGGPMAAw9zE8u/cUWW2zCe/vvvz9Q7lbW5/7rv/4LiImkCl1Twi4MVop2itTW9bzmmmt2\nnk8lZG8qjCC8T9TK9TwbKNwivY7Q7Zq+7bbbBjl0bXV2unzuc58D4KMf/SjgMD+R3v8qd96gzXVr\nJ9fTTz8diCF3UF56dzK0NYJCAvfYY4/Oeyrpu8suuwAj2ZC6dnItQ2PLTTfdBMQQqEceeaTTRmXj\np4O2tEg3+FW48BRphFxztEm8CqlADPPTRr5lc7UZpJFy7Uce5qftDtLQ1hkoR1+rudaf//xnIOpj\nikL/VGjq4YcfnvR46qOPOOIIANZZZ52qpzIMHOZnjDHGGGOMMcNi6KXRtbrUZnwAxx13HBC9TlXQ\nCvbzn/88MCPeqEagpFGIBT2qFPYQ8jrJ25Ru9qlE9HEsz1sXZCndbLPNALj66qtn83Qawbx582b7\nFGpJ6jGei6XMh42SotOS0dqc95RTTgFg6623BmIp+hQllW+00UbA4GWA5wK5B/VjH/sYAI899ljn\ntVtuuQWI84YbbrgB6L1ZOkQPv7zYKuix9tprT/+kG8gWW2wBdG8roeJHs+yRGltUrOaf//mfgVjY\nZskll5y1c5ptVlppJSAWkDjssMM67x1zzDGVj6OCSocffjgw4x6pvtgzZYwxxhhjjDEDMLLS6Olm\nhF/5ylcAOPXUU4EYq7v++ut32sgqqBLo73znOwFYeeWVBz2F6TB2cbw1oVZxvLOJcid+8YtfAM6Z\nqsKDDz4IdOe0fOELXwCix2AEjL1cZwnLdTRYrqPBch0NlutoqPVcK113qD7CQw89BMTtY1ZYYYVO\nmwMOOACIOaYLLDCrfiDnTBljjDHGGGPMsBiZZ6rh2FoyGmptLZlJPvWpTwFxc2t5qgA++clPDnJI\n6+xosFxHg+U6GizX0WC5jgbLdTR4rjU67JkyxhhjjDHGmGHhxZQxxhhjjDHGDIDD/Mqx63k02PU8\nOqyzo8FyHQ2W62iwXEeD5ToaLNfR4LnW6HCYnzHGGGOMMcYMCy+mjDHGGGOMMWYAvJgyxhhjjDHG\nmAGYLGfKGGOMMcYYY0wJ9kwZY4wxxhhjzAB4MWWMMcYYY4wxA+DFlDHGGGOMMcYMgBdTxhhjjDHG\nGDMAXkwZY4wxxhhjzAB4MWWMMcYYY4wxA+DFlDHGGGOMMcYMgBdTxhhjjDHGGDMAXkwZY4wxxhhj\nzAB4MWWMMcYYY4wxA+DFlDHGGGOMMcYMgBdTxhhjjDHGGDMAXkwZY4wxxhhjzAAMZTEVQlgrhPC3\nEMIVyWsvDyHcGEK4JYRwaJ/PviGEcF0I4doQwo96tJn0WCGENUMIZ4cQLg8hXBVC2KP9+g7t418z\n3d85asrkmL0fQgjHtOVwVQhh65I2S4UQrkj+HgghfGWS712+LbvHQghf7dPuqBDCDe3vPimE8Nz2\n67WWcQW5bhhC+F0I4e8hhEP6HOdlIYTL2nI9P4SwboXvPrB9vYoQwgqTtF06hHBneg2S6zJvsu+a\nDSrI9vAQwp9CCI/1OcbCIYTvhBCuDiFcGULYqeJ3/0sI4eb237/0aLNlCOGi9jW7JIQwv/36Pu3r\n8ssq3zXTDKMvyNqfUuX+rHrcXtc1hPCBEMIf+/UjdWHQcavXWFPS7tchhIf66VgI4YC23qtP2bj9\neq371F4MS2+r9BtZ+11DCJe2ZXlpCGGXCp9pjK6KXL4hhDXauqg51L9P8vkXhhCeCSG8rsf7VfT/\n+SGEM9vX75wQwurt19dp63GlazabDFFPf90es64NIRwbQliwwndXGRM759f+OzZ5r9ZzgpRp9LEf\nbOv0VW1de36PdpXWGe22rw2tedi89v/D7WOLopj2H7AWcE3y/4LArcDawMLAlcDGJZ9bD7gcWLb9\n/4olbaoe67+Ad7efbwzc0ev86vo32XkCewD/CwRgO+DiCse8FNhxkjZLANsDBwBf7dNuN2Ch9vPP\nA59vgowryHVF4IXA4cAhfdrdBGzUfv4e4LsVvnur9vffAawwSdujgR/l1wA4B5g323IcULbbAasA\nj/Vp817gO8m1uBRYYJLvXQ64rf24bPv5siXtzgBe0X6+B3BO8t5OwC9nW4YDyrVyXwDs3darSe/P\nqsftd12B/fr1I3X5y2XMEMaarN3LgFf10zFg6eT5q4FfV9WBOv4NS2+r9BtZ+62AVdvPNwXuqvi5\nRuhqL/m2ZbR1+/lStMaoCTrbfn9B4CzgNOB1Pd6vov8/Bf6l/XwX4PvZ+5WuWZ3kWPJ+VT1duv0Y\ngBOBfSt8d5UxcbLzO4eazgn6/Y4p6NjOwOLt5+8GflzSptKx2m2XAs4DLkrlNsw+dlRhfvOBW4qi\nuK0oiqeAE4A9S9rtD3ytKIoHAYqiuG8axyqApdvPlwHunuZvqCN7AscXLS4CnhtCWKVX4xDC+rQm\np7/td9CiKB4viuJ84MlJ2p1RFMUz7X8vAlaf0tnXlKIo7iuK4vfA05M1ZYo6VhTF5UVR3DFZuxDC\nNsBKtCb/Y0NRFBcVRXHPJM02pjXIqw94CJjM6rY78JuiKP7a7j9+A7y87BQYz36hUl8QQlgS+CDw\nmWEet+J1bRpDHWuKojgTeLTfFxZF8Ujy7xLtY48zI9Gvdj+r63AtsFgIYZHhnHJ9KYrinqIoLms/\nfxS4HlitR/P30Zrwl82zoLr+d/pr4OwebZpOVT3V/bsQrQn9pPfvmPadVamkY0VRnF0UxRPtf3vN\nNavqK8CnaTkA+s5xp8OoFlOrAX9K/r+T8ht8fWD9EMIFoRWKUzYZqnqsTwFvCSHcScvy8r5BTrzm\nVJWF2JfWin4UA/S/0rLczCXeCZzW1rG3AkcM46AhhAWALwI9QwzHnCuBV4cQFgohvADYBlhjks9U\nvRfeDxwVQvgT8AXg/w3hfOtA1d//aVq69UTJe9M57jgyK2NNCOG9IYRbgSOBg6ZzrAYwE/r1WuCy\noij+PuTj1poQwlq0PHQXl7y3GrAX8I0+h6h6ba6k5e2mfcylQgjLT/2Ma01lPQ0hnE5rgfoo8LMh\nnsMLQiuU+NwQwg5DPO5sMsj9/w7K55qVjtUO0VyjKIpfTe1Up8ZsF6BYiFao307AG4FvhnYezgC8\nkVbY1eq0XLTfb09S5zL7Av8z7IOGED4KPAP8cNjHrjkfAPZo69h3gC8N6bjvAU4riuLOIR2vafw3\nrY7wEuArwIXAs0M69ruBDxRFsQat6/ftIR239oQQtgTWKYripNk+lzFjqGNNURRfK4piHeDDwMeG\ndI5zkhDCJrQs0P822+cyk7Q90CcC78+8neIrwIeLovjHEL7uEOClIYTLgZcCdzG8/rpxFEWxO62w\nvUVohT0Og3uANYui2IpWZMGPQghLT/KZsSOE8BZaUSpHDfj5BWjN0w4e5nmVMarFxl10W5ZXb7+W\ncydwSlEUTxdFcTuteN/1BjzWO4CfABRF8TtgUaBvwn/daVsslYC4KtVlQQhhC1r5TZcO+Zz2A14J\nvHlEHq+RUyLXKp95HrBFURSy+v0YeHFJu9Pbx/3WFE7pRcCBIYQ7aHlP3hZCGIrXqwkURfFMURQf\nKIpiy6Io9gSeS6sv6BBC2Da5Zq+m+r3wL8DP289/Sis0oHEM2Be8CJjX1qvzaUUBnDPJV1XuY8aQ\n2R5rTgBeM4Tj1IbpjGEDfNfqwEnA24qiuHUYx2wCIYTn0FpI/bAoip/3aDYPOKHdF7wO+HoIIde1\nStemKIq7i6LYuz3R/2j7tYem9ytml+nqaVEUTwInk4WZhRAWTI57WNXzKYri70VR/KX9/FJauUHr\nV/9FtWUqc9h/oqVfr+7hZa5yrKVo5VCe09b97YBTRlG8Y1SLqd8D64UQXhBCWJiWh+SUkna/oOWV\nIrQqna1PK5F8kGP9kVbCLx5WGjsAACAASURBVCGEjWgNcPdP/6fMHm2L5Zbtv7tp/e63hRbbAQ/3\nib19I0P2SrXDMP+DlnJXDRuqHSVyrcKDwDLtPDSAXWnFp+fH3r193HdO4XzeXBTFmkVRrEXL6nd8\nURR9K9OMEyGExUMIS7Sf7wo8UxTFdWmboiguTq7ZKcDpwG4hhGVDCMvSKo5yesnh76ZlPYWW1fDm\nkf2QETJIX1AUxTeKoli1rVfbAzcVRbHTJF81lT5m3JjxsSaEkBoP/5mG6mcvpjmGVaYd0fIr4NCi\nKC6Y7vGaQggh0PK2X18URc9IiaIoXlAUxVrtvuBnwHuKovhF1qyS/ocQVkg8sf+PVmRBoxlET0MI\nS4Z2HlUIYSFa9+8N2XGfTY77iarnE0J4XmhXBgwhrE3LyZDPjZtIVR3bCjiO1lyzV47fpMcqiuLh\noihWSHT/ovYxLxneT2oxksVU0SpScCCtyc31wE+KorgWIIRwWNuyTPv9v4QQrqOVyPghrcYHONbB\nwP4hhCtpLSL2a6rnpA+n0bqhbgG+SSs8DIAwscTnG5jCYqq9av8SsF9oledWid5vJav4r9Ja6f8m\nZOU6m0wIYeXQyn/4IPCx9u9fuv3eaSGEVdt6uD9wYlvH3gp8qMKxD2ofe3XgKnmsQgjzpui9aiQh\nhCPbv3/xtlw/1X791YmlbkXgshDC9bRCnd462XGLovgrrXyg37f/Dmu/luvs/sAX29fss8C7hvfr\nZpWp9AVDP26v69pkhj3WhBB+S8sb+rK2jHYvOdaBoVVW+Qpa/U9pif8xYlT6dSCwLvCJED0BK7aP\nlfYH48ZLaPWXuyS/W9vCHBBCOKDqgaag/zsBN4YQbqJVNOnwof2a+lBFT5eg5eW4CriCVt7UpHOi\nimPijrTmC1fQWvweoPGtyUxBx44ClgR+2tbpCQuuKRxrRgjDWG+EVuLjL4ui2HTaBxsBdT8/0ZTz\nLKPO517nc6tCaIVmHTIKa8p0abJsQ2s/q0OKonjlbJ9LTsPluh+t8rMHzva59KPuMq77+ZXRtHNu\niq6KJsg3hPBYURRLzvZ59KMJcuxHnecEKXWX8zDPb1ieqWdphUBNxyI6EkKrCsqpwAOzfS4VqK0c\n+9EAGTdSrtDaoI/WPgqTlW2fLRop2xDCPsDXaYVv1pGmyvUDtEJ/ypLg60ZtZdyAPrUXtZVpTsN0\nVdRWvqG9aS9w72yfSwVqK8fJaMCcIKW2ch52HzsUz5QxxhhjjDHGzDXmeulwY4wxxhhjjBkIL6aM\nMcYYY4wxZgC8mDLGGGOMMcaYAfBiyhhjjDHGGGMGwIspY4wxxhhjjBkAL6aMMcYYY4wxZgC8mDLG\nGGOMMcaYAfBiyhhjjDHGGGMGYKF+bz777LMFwIILLjitLxnGxsBlx9BrIYSux370a5Mcb/IDTYNB\n5DoVGQ56+tO5TnWQK8A//vGPAmCBBarbCXI9Sl+r+J1dj7quUzmHQdF3LrDAAiOVbdEWyAxcwoH0\nsOwa9qJKm5nS2UH0tYwqMsvb5DKbSh9bJpYqopopfZ2NsavX768i1+kcPzvejMhV+jrssWa6pz+M\nuUZ6HjMl12H3rzrvfrrX6xpW+Uy/7xQVf0uj5FrxO7v+r3Lf9mMq5z5T/SvMbB87mUzT93Pdn8qY\n1U/Wk8nWniljjDHGGGOMGYAwyapwOGae5jHqVb3lOjpGKtv0fpEV49lnn+36fyqe0iFinR0NYyfX\nXh6qstdyHR6it3Xs5FoTLNfRYLmOBst1NDR+rjUoo/J6p4cqe9GeKWOMMcYYY4wZgL45UzPJMGKd\nZ9gTMBbMRC5WE5FcnnnmGQCefvppAP72t7912iy0UOv2eeKJJ7r+X3zxxQFYeOGFO23z+PRclorH\nhenHIJu5R34fS5/S16XLepRHVbqtR4j6Kb3NdTr1UElf51L/YIyZe/SbL/XKPR2kXyyLgJkrlMk4\nnR+VtZ1KbmA6dg1TtvZMGWOMMcYYY8wAeDFljDHGGGOMMQMwo2F+uUtOYSZl74mkHOGEtnnIVL+S\n1OPoKs1lJlmlcu1VQrJMrvlncrfoXJErRBk+/PDDADz22GNdjwDLLLMMEOUi/asSOtmraEX6+XGV\nbVXyULUyWc1lGaV6lt/7CuVTCCrAo48+2vX4yCOPdD0+9dRTnbbSZen4csstB8DKK68MxLA/gOc8\n5zlADHMdx2uS95tp2MlkJdGrjF2zVLRm1uknV+lTL+aarEZN2RxMfYKuRb+5wDjTa37abzuV6Yb3\n5a+Nq773msemYed6rvEtT8NI0eclL+nuYostBsTxCibq83RkPLfuCGOMMcYYY4wZEkP3TPVLHtOj\nrB2ykAL89a9/7XrtySefBLq9LGLppZcGonV0qaWWAmDRRRcFYIkllui01YpTq9F+1u06WgDy1XpZ\nQrmsz/KayNIM8Je//KWrjR5zbx5EOcr6/NznPheAJZdcEuhe0Wu1Pw7W6DJP6UMPPQTAn//8ZyDK\nTboHUR7Sw0UWWQSYaMUr+y6Rb/Q7V0nlItmfeeaZQNTh+fPnA7Dxxht32qogwrD1r859Qd4nQOwL\nVCBF+qp+FeCuu+4C4M477wSibt93331dx4BoxVtrrbUAWGeddYCo28973vMmnN8wrXyzQZlc9Vzy\n1PgknQS4//77AXj88ceBKEfJUP0CwIorrghET19uLS0rWtOvwEeTxq60f/373/8OwAMPPABE2aWe\nf72Wj92Sp2QHUZ7ypEqOZf3qVLyBdZTrIJR5sTUP07WQvG+99dZO25tvvhmIst5ss80AeP7znw90\ne6jL5hT5d9dZX3PKNoId1qbw/b5rsjZ1ltlklHn4pI/qN8vWB/k8VnMErRP0CLF/0LxVfcLyyy8P\nxHktxL5F+t0vum0y+dszZYwxxhhjjDEDMLKcqdS6J4tUbi39wx/+0GlzzTXXAHDdddcB0WIlS1Xq\nEZB1b/XVVwdgtdVWA6K1RP9DXIVqtSrvVVNW97l1L12BK5/n7rvvBuBPf/oTALfffnunjazOapN7\nsVIvnqx7skJvsskmAGy44YYALLvssp22WvXncalNRPqZ5ozIOnfPPfcA0XKRWlRWWGEFoLzkZk7u\n/VIMsK6r9LLsOE3R1emQ9hcXX3wxAN/+9reBiTk+a665Zqet+oVc/3qVqS1rM9X3ZpsyD4p0V9bl\n3LMKcNtttwHwxz/+sev/e++9F4gWaoh9rPRVFrwyL3XukUo9MU1Cck37gdxaf+ONNwJw9dVXd9rk\ncpWFVf2lLKIQ+1L1saussgoQx6yVVlqp0zbtm6F5UQD5mCUPHsTxXvLU2CXvKcQxSnqZ9pEQZQew\n+eabA7DpppsCcS6gPjrVScmxSkn/psh6EPT7dY/L+n/ppZd22vz+978Hoida46D0NfU2Sq79clxF\n0+Sae1TKPCyTbX9Sdrw8akuP6Xg2jnmVZesDRVVonNecFeCWW24B4ppBjxrf0uuQ66jmC+pzFXkF\ncZxTVFbqtRJV8wPtmTLGGGOMMcaYARi6O6EsPjr3SMmSd+2113baXHnllUC0+MmyKgt+GksuD4qs\nWS94wQu6vqeswoqsg3muD9SzMk1u+SjbNFarcllNJTut4iF6+PQoueo4qRxkxZMnQJbR3CoFE2PS\ny/Ik6hofncfoygKaWkKko3fccQcQf+8aa6zRaSNLqeQjK3aep5e+llZXg3Ivg75Lnx/nfKoyb4Cs\n1dJRWUzlJZQnBaLFuaxyWv5/r8qWddHLXvSqgpr2sXnsuPpayQwmeqTkwZbepn2s7g31BQ8++CAQ\n+5HUS53nCkpfy6yrdZR5nkOS3qMaY9S3ymsqWUK0kqa5qhAjByQziN4ZfV7ef8kl9Ubp/tdjmZe1\nTnIUkqc8SoqOSMf7Cy64AIArrrgCiDqZRl5Inr3uW1mTIeb2SNbbb789EGWXWpzzSIIq+Sp1lPNU\nSM8//9352JTmA+oayAuo+173eDo2NTVXUuQV4sr6V/URGpvKPHPqR3t5ndLP6bj5fEoRABDnXXl+\ncJPknI9dqSwkS41DmoeluXuXX345ENcJmuNq7pbqoXKk5OVWP6zrWlYlUF4sybgsF9A5U8YYY4wx\nxhgzAobmmcot7KmlWdYNWZS1Ykxze+Rl0ep+1VVX7Tp+mYVZq1xZYeV1SSsC5TlXedx13cmtGulv\nk9VY1k7JIc190Odk6ZDlUzJMV+m5105WE1ms0ms6FetIXSwo0lHJR4+yxKV5EHou2ebV0iBa5HNv\nk3SszGql4+izsqik1yH3oo4zkpGsUhAtS0I6nFdWhNi3yEotuU6lGk9Ztak6kp9/WdU53bPSbVnl\n0ufSd8lI8k49z9Lh3CsiPU2vl/qUfA+QKhb/OpBbpNPfJp2TZy7PIYMYk6/7VWON5Jv2Gbou6ldU\nWVFVE9M+NpdnU9D5Shd1j6b5e9JFyUPRJmmfl+fp6fqUVfCSJVl9g66Xxsg051rk93qd7/1hkv9O\n6al0UdZ/iH2Fon822GADIMp7HKImcm+J9CrtOzVnlZdVHtC0WqqOk3voyqKA9F3y9CkqSJEvabVU\nfS7tc5pCrxyzdL6Tz8P0KG81xAiBNO8Syium5hES6n903LQvUFt5rjUWpmNrVR23Z8oYY4wxxhhj\nBsCLKWOMMcYYY4wZgJGF+aVuvDzMTy741JUmt6bKFsrdqeOm4W36fJ6IWhbak7vomuLK71VKO91Y\nMw8DyRMgISaOyi2fh1ClISg6ttygebhGKt9eyaZl8q1L0nme9Cl9VJGJ8847r9P2hhtu6Gord/L6\n66/faaNwPOmfdE0y1XHT40jG+oyOkZb5VahP00JSp0LeT6Shqfnmjwq7kgzTcIl809O8THdZiFT+\nWlnRlDrSKzwnfS9P/i/bSkHyU18rHUyTbiV7hfDl5c7L+gLRr1BCnULWehX2SMcu3Yv6/Sq5nYah\nq09VqJrkqTFLxRHS5xoTJWddt6boYj96hfakBSM03uv3K9wpHZ/0Wp7QrxArhaVBvGbS4Xw+UlaA\nQTRVzoOS34PS00suuQToDnfXdVIYar4VRZNll4f35qHRaSEvFUyRzincLw05l+5p3JaspJtpgRq9\nJn1Xaf90G4X8PJssa1EWAp4XTdK8TGk7EOdUkrHCItWnpHMCyTZPbdFcIW2r4w2jT7BnyhhjjDHG\nGGMGYGieqdyanCbLaSWoFaMe0zKwWsXLuqfNtPTZtBStLCmyCmiVK4tAmXVL7+l4ZVbdOllN8/Kt\neQny9LV8FZ2W1ZQ8ZYWWzHWcNNlaFpq8JLfk0q8MahM2PswtULKEKOH2oosu6rSVdUS/fbPNNgNi\nIi5E60i+oa8SHdMCFPJSqeynZCyLVCojWUvS6ziu5PoNsUxpLx1L79PUu13Wts6e0qr08kiVeYfy\nIitpgrSS8WXpl5dFVue0AIXuFXlmpJPqp1MvQ9639rNa10nmeR8r71M6LmkTXclT92Sqr2ojK76u\ngSIobrrppk7b1NoKE2WWegAl17Lr3eu31AHJRucvmaXjsop2qA/VeJ+WMJfO6VEeg7LiSZoT5Nen\nTHZVNusdZ/LfraT+s846C+juM9RHqPCEruk4yC73WuZRUNryAKJHWY/ypqT6uvbaawNRVrqX1Q+k\nc1h5tiTrfPPZtH/pt/1M3ek1dpUVdVAfqzlpWXE09bGKXFO/kc4DJO98Pqu+Jo0q0LxYfVM+Py77\nDb2wZ8oYY4wxxhhjBmDotZdzD1X6Wm5ZS71XspZqZaiVqFap6UadiluVFUpWUlm0U6uprAN6bNKq\nHibmQqQy1MpdHjo9pm1SazPE1brkncpK7+Wb1pVtdpafX10oi5EXeVlOWYUUK56W7pUFWZa5+fPn\nA9GSAVF/81L90t1U9ro2smil+VQQvQYwceO+Mm9q08ljp9M4Zt2reYyzLFdp3mBurcuPW8U7Ujcd\nrkp63nlZX+mvLKAQZSLr3nrrrQdES12qX7o3dK/ou6TzZR6U/H5oilcwH7PS3DFZLJXLUGZZzUsg\n6xrIEq38S4ibKCtHUp4+XYO0z+iV/1d3ueocdN9KnmleqO5l9Xtl20lInvJIyYOiuUC6uWyvvjfP\nZy07z7mGdEVj0qWXXgrEMSmVyxZbbAHECJemySyfN/XLoc3lknropGs6njxSilgB2HnnnQFYffXV\ngTjH0rwilZ28J3rU8RUJMK7bopSNDfmWFLrvJZu0vcYueQHVb6RbSug4up7qu/VZebMg6rX6+bII\nsKqMz+zMGGOMMcYYY2aQoedMlcUoa5UtK5FWhqmVQJYlWQVkEZU1Spt2QbT8yQKgmH95prTaTL8z\nt5r2+w11pCxOvNdmZ6kVTqt0WQdlUclzICBeD107fZdiiNPKYFVyU2aD3ELbL79GlifJLc0fk5x2\n3HFHALbZZhugW16yfuZWe1lL0k3/8g2QZaHWOaXXbJ111gFi9R95qMah0pfILYZpf5FvTig9lzzS\nCpQ6Tm6ZrpteDpOyTXul57LQSb9SnZalVB4CPUq/Uku/rIKSuT6rx7TSpK5XlQpfdbweuWcqrYaq\ne1y5t/qNqccvjwxQtbnLL78c6PZCS1byCipnsswzpfOoYiWto1x13mUba+q3SW8lz9RDLznmHir1\npWmfqXFeuRR5hcqyfN+5ju5xVbHVOK8cQIA999wTKM8laRL9Koz2yq9LK/WprTwYG220EQAveclL\nOm023XRTIM4R1D/neacQdVnfoX5B90U6Bo5Djl8+HpdtOJ+PYelcV897Remor4A45mkdoHWBHlP9\nzmsq9NugdzL5N/POMMYYY4wxxphZxospY4wxxhhjjBmAGclykwtNoVNy5aelJxXGlxc9kPs/dfnJ\nNadQCZWiVLnVtKR0lfC+JpCHM0EMgZQcJcPUVZm7j+WmVlhEGtKi78hLfatNWRn1upNed8lCYSP6\nXQr3S9uq0IR0SwmPqW7lYU0Kk0g3mBZy8Sv8SqEqd911FxDLdqbv6T6RrMchKVU6lutz6vbPw8Z0\n70se6eaHeVjnOIf3ibK+QOEi6i9Vgj8NNc3LT6ufULiPdBFikR+FXCh0pSzMZ1xCK8sKJSm8KQ9L\nTZPTtbmn5Hj77bcDcN111004nsL65s2bB8QxTMnRaXhPU0OqRB4+lf62vI+TzNIy8npNbXWvK1ww\nDedfd911u74rD0ttuiyHicaT2267DYhhfupPtt56605b6Wc6T2gSg/RJkkOa1qC5j8bpDTfcEIjz\nAoh9pPQ+L/WdFrjSc70n+Wp+kd4r4zKHhYkbpEMc3zXWqHiSxrC0jeSkeZSOk245odfUp2pdoDC/\ndA5XJbyvKu5hjDHGGGOMMWYARmbqTi1BWv3lJXTTBL9bb70ViNa9fHNDJZYCbLzxxkC0oMg6IK9L\nakVpqkWqlxVC1g6ISXdKHNfqPU30VbKdvHn5RohpScl8IzlZDeXJWX/99TttVSShThsdQ/+N7fLk\nRXnyZIFKrSVKzldpdFmo0w09802AdR2ku6lXQK+prXQ/TUoVek8eGH0mva5NtVLlSb9lpdHlBZA+\nS865pxQmbn3Q1Pu9Cvr9eWIzRF2TR0mWz7S/kMwlT3mipLfysEC8N+TBltW17HqJpupkTmql1P0p\nXZQlOfXSqwT6NddcA0SPlOQsbxTEhPWtttoKiEWU8g16xwnJM7W2SzbSU3mkrr766k4befjVL+db\nRqTFgKTDitLQ5uoqRJFu1DnXke6eeuqpQOwzdH1e/vKXd9oqQqOpelmlGE6vrTLSvjPfciX3WEMc\n4zVfynXy+uuv77SVzKXDedRAOtaPQwEKkY9hEMexfLuEtDR6XrBDsi4bC1WQTrLVo/qNdM41TL1u\n5h1ijDHGGGOMMbPMjCRhyOKRx4OmJY5lLdEqXqtSxUWnHgF5RRQLKevTOMVH98oFKSuHLOux/lc8\nL8QSnvJQyfKh46WbIcs7eNVVVwHRUi3rYbpBnbyCdfNM9SO3isjaIctnajlVG3nppI9p7l4erys5\nycOX5qDIyiqd12cVg53mTMnKkudgpbG+TdLxVEdyfem1oTdE63Ve0j4tH61r1iR5TBfJJfXs6z5W\nGW7Fm6dWOFk45dGWR0W6rf/T46m/ULx5vnUFRF2u08ax0yE9f3ng8k1l05xReZDVf+r+l1Vf3m2I\nuT3yfEuX+3nUm47u0fS+lbU4z1lIPUiShWStyBN9Jp0/SObqK9VvK+pg11137bTVNR1GnkRTSPvQ\nm2++GYieKcl3k002AWC33XbrtG1qrtRUUL+lPk3jedrH6f6UzuleT6NPJCt5WKR78kKlWyToOOoH\nlN8zzByeOpHPvVJ9zOcAZXlVek33tXKmRBopobmUjqt5k67PqGQ7d2YgxhhjjDHGGDNEhu6ZKrM0\n55YgrTjTlb8snvJQaYUuS5ViSmHiSnMcrXk5su6llmZZjeWhk5xTD5I2kstzHyR7WU8hylVVlHIr\ndGpZyb0GdbFKV/n+PEeqTGflIZWXTrKVrFNyy77yTdKYa5FXRdP1TL9bn8vPL7XU5NaV2ZZ7P8rO\nLfckpZYl6aEs2XnOmPL/YG5ZTnOrXlrVUNZUeZT0XuoNUCy6+tjcW5pWr5J3VJbXPA8w3Vy1V39c\nZ50so5+e6hpIVrI2Q+wXJSP1tYrdV+RESlnuwLiRV/GTXCDmQKsirWSnXCeI97n0S+Od9Dat9nXF\nFVcAE/tg6XHab2+55ZZd59c0PZ0K0rO0rzj++OOB7mrKAPvuuy8QvSXjjmSTR0tIX9M5p+ZJ6veU\nF5mOP/pcPk5prE7rAKgPVv+syKGy+UB+nk3W13xjZIjykiwUhZZG62gOoP63XyRKPs7l8htVNJU9\nU8YYY4wxxhgzADOSMyUvhjwcincsqw2vHClZqPI9USB6AvL8k6YwHQtDasGXVUQWEFlPVO0QYry+\nVv/6jK5J6unSSl7H02d0vmnOTtn+QHWgn2z1mn6z8kCUx1Dm5dBeHHovzVPRtZAlRO/JGppWo5El\nJfXGQpRxmhOo35BXvGuanpeRX5eyPWhkwZZ8Jc98r7Sy4zWNKn1BXrFI+qZqURAt8bL058eHifun\n6Li6r8v2/si9pPIKpNXs1CavSNf0awMTZaaKfTfeeGOnjbyB6je32WYbII5dqRVVnhP1PZJZXmkU\nxkN+MLGCJ0TZqO+VLqbjkXKjpZ/qK6Vvaf+qPuP0008H4NprrwViBTVFGKTfqT5n3PJTUnQfX3rp\npZ3XTjzxxK73dC322GMPoLsvHmfy+0t6IHmk+20pokR50KlnXmhOpWgqjVPyAKaef93vGvfVH+j/\n9Nzq2g9UGbtyb1DZHol5pUTpn/pGiF7CvK3WAOm8SnJWFIXGzbJogGF6++yZMsYYY4wxxpgB8GLK\nGGOMMcYYYwZg6GF++YabEMOflKir0pwqdADRtfeiF70IiMm7CgVMXfoKaVGZRCX4pm5BUUcXfhWX\nYh7aJRmqrDFE96Xc0kocTRP3FDahEIm82ELqbpWbWu5RuVAVeqnvgYkhaE1IjtS5KdlTG2jKtX7W\nWWd12sqdr7Acya9ssznJTTJRGIrkBjFZOk/gV/hAGmKojTzzbQHSNnWWcxV6hftB1EnpunRLoWVN\nCIEYBnkfoN+vMtDptgZ6TTopnUnDpvIw0TwcLw3dk36qjcJSyuSdlxZWX56GGDbhOpWFgehe1lil\nxHMVPEjb77DDDkDsVxRuomsDcezKw3rKNrYt2xi5SeTyTOcEeUiP5JFuxKsiFZJRLg/14xB1W/LV\nth7SaZWyTl9TaGAd5wjTRTLX/Onkk0/uvKe5lMbzd7/73UCU91xD1199peae6TxKY7J0J99oFuL9\nLj1V2xtuuAHoTmvRfEJ9hQqkNGnz7qnMYzVH0pwyLYiisUav5QWnIPYBaaoJxL5Vm36n36njqt/p\nl5oyjPlr/a+YMcYYY4wxxtSQvqavfqu1XollWgUqURliEn++GWy68pw3bx4Qk0NVJlGr+rQMqj6v\nJOB8096ycpWyPtTBQlpFrlpFa5WuVXYqVyE56jemljYdJ7dKy7KSWku0upf1VdYEWfByqwBMLI1e\nZo2eSa+VzqfM2qjvl/Vngw02AKJVNC0Je+655wJRFvpsWh5eeqZrIkuf5JQmnGpDX31eMpHep2VY\nVTRErzXJWjVVyspoS1fTTTkhynecyqFX6QvUB+QbFqaeKema+k3d3ynybude6bysevpd6lvVx6g/\nTb0EuSVR/UY/z0qd+mGhviO1Nuu+vfzyy4E45qRevBe/+MUA7LXXXkAcw+QBOPvssztt5TmRDude\nvNQzI0t5P69gHcnHhLwAVfqeHuWFT3+/+j3JKi9Tn3rx8giMvIhCek1z78I4bfyde/HlUT3jjDM6\nbaRH66+/PgA77bQT0HxPaBlV5h65l1RzhzTSKd2OA8oLKajP1HxA97r61fQcpO9bbbUVEOdYTbnH\nIcqg7L7J1wMae9QnSlYQ5/b5xtvayBjiHE33t46Tb60A8ZrrNfU/+WNKlXXCZLrU/N7DGGOMMcYY\nY2aBvqaIqcREysojC2ma26PSkNroUG0UmwrRS6DYVLWRdTotA6zz0opfq9N+eQJa0TbFypfnSime\nVDk3EFf3aqvflm7EK5noOsmKoONddNFFnbYnnHACEL0xOt56660HdHtu8lK+o94QrSpVrIv5JtKy\nfCp+GaJ1WV5QySSNzZWOynovfZaupjoruegzsrToM+mmkrJa5SXqx8Fy2ovUWiRPtqxbshCqVHJa\nnr7pVCmJnseZyyqX9gV6rr5QfYM8KxD7CclaXiwdP/W2SF+VR6H7oazc91R+Ux2RPCSfNJ5fnmTl\nTspqmo5d2iQ99ySXbd0hT4FeU/+sa5HeB7r+aiPrad3lm3tUZZ1O9VVjt7ysvXIiYGI55DIPouYY\nknke4VGWk5lH1TQ1FzMdcyVr3f+//e1vgW6dlhdk/vz5QNRljYtNy3WcLr1KpJdFt+Tzm9TbqnFa\nY7zeU3+dev7Un2pM6Zrb8wAACitJREFUyzc8bwJVSqJLBoqskrdI9336XI9qm27ynR9X97m8WWnf\nIu+2+hTd3+pb034jnxeXjWtVN6Ef39mZMcYYY4wxxoyQaQfJ5rGjWlWmFlHFjmr1qBWnNjeDaM3L\nLapllVAUS6mVZr5ZahovncfD1p08zly/Sb8xtR5LnrI66TG1gMhKIjnoONrM8Hvf+16nrbwu+vzm\nm28ORI9Nao3V9cqrA5ZZ92bbW5WTWxr0mFpFZSmWjsqCJE9V2iavDqMY4DS/Sh4DyVDePsWty1OV\nnsc4xfL3QrJLq3VKr3Wf6zHXYWhGFcnpklchkrWvrC/IIwRSucoiJ72SnsmCJw8rxEiB7bbbDogb\ngcs7nea75nlUZXHndewLJE89Sq/SjY9lLZWs1R+klb40nqmN+lFt8Pt///d/nbaK2Mgri8pTk+YD\n5rlBdY+q6DV2aU6Q5uQp90zjfVm+Tp5Hpv91nZTHBnDZZZcBcbPefKP5tddeu9M2H7vqKs+qpN5M\nWezlSb3wwguB7jFKOrfRRhsBE3PzxolhX9vcO5rmRfeqjqo26fxiww03BCbq4rig36OxS2NWWX6u\n5lSa80uH03mPqitLpmqrxzS/TWNVXg1Ux0vvhTy/czp9wvjO0owxxhhjjDFmhAxsisg9ErnVKM2Z\nylecWp2mOSXyqmglK8uVYszTlacs1bKSKo5fFpa0bV4JqA4WgCqVZXIrb76nEUSZyZKqnLR01S+L\nneQqb4n23ZDXMG0jr8nOO+8MTNwLAXrH8Zf9tpmU+SDfVfaZPH5cuXxp3piujawuuh7SvzQ2V1ZA\nyUvWVh03tfTru8fRUijyXEtZryHes8pBkTdPHj9ZqSBaoZpKP31Vn6V+TZ7QZZZZBuiOKVcfqHzJ\nssqmssLpeLl8U92W5VQeWbXVd+tcYGLuYVPyqfKxS+NTWmlKY5f0U+Nbem+r39W4pvwd5VnqfYgy\nkvwkO93raQSFLNl1qkTbj3zsknxlTU7lqjEq3x8tnTeoj5QXUJ/XvEFRKxDnC/IkSmZbbLEFEPPa\nYGKFyjrNDaZC2b5oknWep16Wr7PpppsCEy33TZPDTJJHB6Q5U5o/ST91DfJIC4hzLPXhTYw+mUqF\nRFFWU0F9gF6TvFKZKFdVn1c/rOuhCB+I1VU32WQTIOag635P51p5/+t9powxxhhjjDFmhvFiyhhj\njDHGGGMGYNpxRHlCsVydqbtMrr7c3Z+WM1TYhNx4elRIioohQNzoVEnSSgpWeFXqxpOrsE4hU1PZ\nSE6/RWWyFWYDMQxCoQ1KfE6LdeSJ6XmxhDRUSKE9++yzDxDD/BQGpHAAmFgcYVxDA/KQu7LfKb3T\no1zHZYmOkmFeGrksHKsKTS3AkJf+T4tK6LdIHxUGkBcKGAf6Xb88VEKhdgr1TUv3KsxP977u97R/\n1r2q/lK6p74l7QvUz+Qlq3ttigr16mOrkG/nULapo2SsNgqpTsNSFbImmev/tB8WCpVWqJXCfRR6\nlm5aq1DApm5SLX2TDFOdkR5pTqAwvbSEt8Yu/f5czmV9hkJVtRnq3nvvDXQXT9LxykpfN4lcbyHO\nm9RnijTETP1qvvWGSEv5N+2eHjV5wbW0f83DhPW/7u10/pSGSafHnW6430zOB6p8l36P5kZlfVpe\nWEprgTSUWuGU0lWF/qqYitYEENcKSqFQn1o2duXpStPBniljjDHGGGOMGYBpmx1y66ksmunGsbLC\n5RbVtHy60HH0eSVJbrvttp02SuZXOWlZVJticapiNZBFSBZhrbLTVbW8dnpPliaVPYdYcELWPCXj\nqWykNu4D2HHHHYF4vdQ2LzYB4++RGqSEs2Sia5aWO9dGtJJhnvycWmpkxaliLWmC/FNZ5oUnco80\nxPtXFiV5UtQnyLI6DlS5ftIN9a36X1Y5iB4OWUwl39SyXJaAC1En034zL4CTe2jTtv02PKwzecGE\nshLxKqmde5JTy790OC2oBNELpSIeEMezLbfcsquNPAepxbrfthN1Jo+q0G8r6wfUV2puoNLmEOUp\nb5V0Wv2jvLEQ9X+XXXYBYkn/bbbZBuj2ujZl8+PJyEv7p8+lw1tvvTUQ5woQi/bo+uTHabpcRkE/\nj5TQfSoPi8a0sv5aep8XEZmuZ6luxb40Tuj+01w19aaq/5W3Kd+cPj2O7nn1qfJEp32s+hvNX9Wn\nlo1d9kwZY4wxxhhjzCwTJrHAT2qe77VRX7qhpFaYsjTJW5KWR1R7WY1kjZaXRJ4UiNZBWajyUqxD\nYNTL+8pyzS0XaTyzLB/5Sl5lJKE7th+ifGW5kjcLovUgz/3ptwnnFC0hM2E2GcquoL2sdP1+b55z\nkcpeninpvvIBlPeXbiqpa5NaEysw6zo74QNt3U11VnJVPLmseGl+iSymuZVOVqh064MZsMTNulzz\nvqDMOppa+tL3Uvnkr+V9f9p/6r0876Xf5tw16wsqyzXvW9NYfT1Xn6qcqbQ8v16TDqtPlaU1zdeR\nd1EW1jKvvxhQt2sj1/wxLSOt55oDKEol3RRdpaWVK6Xj5NsAwMRN0CV7WaX76WtFOc+6XCd8oCRn\nqlfp7vQ35l7mKlsajJDaybXzwaR/VN8gWWvupfkuxFypm2++GYDzzz8fiNv7yCsD0YMqL6F0ud8m\nvjXrX6GCbHN9VH+azo1yD7TmBnqEeC3UXypqRf1pOo+VPuc5UmURa8PsY+2ZMsYYY4wxxpgBGJpn\nKj9O+n8eD1q2AW3nhLI49rJV5QzEkNfWWtJ1kAHyevrFRffywgyxQkwtrCWjIrdepRZueQ21SbIs\n3cqZSK1W8hDWzBI1sOU0tUjrN6kP0GPqvZJHLs8ns+W0z0GyvqCf7vTqs8s+3+veb0BfUFmuVfrR\n3CuYjl25h6vX5rUw0cM3Ahol17xt2Wdyz2G/TdaHmQORURu5TvhgIrNcNmXyrFlOVG3lmpLf9/lc\nFqK3RTnr2lRa+ZXynkLM+1VEisb8BnmoYRpe6pR8fVBlfNJ7ZVWW8/XBCPTdniljjDHGGGOMGRZe\nTBljjDHGGGPMAEw7zG/CByqEhE3F/T9LLulGuJ4bSC1czyP74iwUKA1dUzhAulE1TNwcFQYOUamN\nzvYq99qPst9ck1LbtZHrmGG5jgbLdTRYrqOhEXLtFXqWjvEqmKCCKQrtV2hgunGyCqQopD0v19+A\nMGqowVyrTuuDWsxWjDHGGGOMMaZp9PVMFe03p7L6q5LwONeTIgeR65gw8h9cB9nmnpn0NVmy8iTJ\nso3kqqDvWGCBBWqjs70S8qt+vmb3RW3kOsJzYKbOIfmusZfrLGG5joZGeFAaSKP1tV+hNZVPLyuK\nkm+GPiyP1Ez1r+3vGsn6YJDjzHB/ZM+UMcYYY4wxxgyLyXKmjDHGGGOMMcaUYM+UMcYYY4wxxgyA\nF1PGGGOMMcYYMwBeTBljjDHGGGPMAHgxZYwxxhhjjDED4MWUMcYYY4wxxgyAF1PGGGOMMcYYMwD/\nH78PDOdJgextAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1080x216 with 20 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nQaiRvHgwQ8A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bevKHIFn3OIf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CZ3A9-Il3ONu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tY7yqxW63OMG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HhsvI1Oi3OGO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cI7Deufa3ODe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "from keras.layers import Input, Conv2D, Flatten, Dense, Conv2DTranspose, Reshape, Lambda, Activation, BatchNormalization, LeakyReLU, Dropout\n",
        "from keras.models import Model\n",
        "from keras import backend as K\n",
        "from keras.optimizers import Adam\n",
        "from keras.callbacks import ModelCheckpoint \n",
        "from keras.utils import plot_model\n",
        "\n",
        "import numpy as np\n",
        "import json\n",
        "import os\n",
        "import pickle\n",
        "\n",
        "\n",
        "class VariationalAutoencoder():\n",
        "    def __init__(self\n",
        "        , input_dim\n",
        "        , encoder_conv_filters\n",
        "        , encoder_conv_kernel_size\n",
        "        , encoder_conv_strides\n",
        "        , decoder_conv_t_filters\n",
        "        , decoder_conv_t_kernel_size\n",
        "        , decoder_conv_t_strides\n",
        "        , z_dim\n",
        "        , use_batch_norm = False\n",
        "        , use_dropout= False\n",
        "        ):\n",
        "\n",
        "        self.name = 'variational_autoencoder'\n",
        "\n",
        "        self.input_dim = input_dim\n",
        "        self.encoder_conv_filters = encoder_conv_filters\n",
        "        self.encoder_conv_kernel_size = encoder_conv_kernel_size\n",
        "        self.encoder_conv_strides = encoder_conv_strides\n",
        "        self.decoder_conv_t_filters = decoder_conv_t_filters\n",
        "        self.decoder_conv_t_kernel_size = decoder_conv_t_kernel_size\n",
        "        self.decoder_conv_t_strides = decoder_conv_t_strides\n",
        "        self.z_dim = z_dim\n",
        "\n",
        "        self.use_batch_norm = use_batch_norm\n",
        "        self.use_dropout = use_dropout\n",
        "\n",
        "        self.n_layers_encoder = len(encoder_conv_filters)\n",
        "        self.n_layers_decoder = len(decoder_conv_t_filters)\n",
        "\n",
        "        self._build()\n",
        "\n",
        "    def _build(self):\n",
        "        \n",
        "        ### THE ENCODER\n",
        "        encoder_input = Input(shape=self.input_dim, name='encoder_input')\n",
        "\n",
        "        x = encoder_input\n",
        "\n",
        "        for i in range(self.n_layers_encoder):\n",
        "            conv_layer = Conv2D(\n",
        "                filters = self.encoder_conv_filters[i]\n",
        "                , kernel_size = self.encoder_conv_kernel_size[i]\n",
        "                , strides = self.encoder_conv_strides[i]\n",
        "                , padding = 'same'\n",
        "                , name = 'encoder_conv_' + str(i)\n",
        "                )\n",
        "\n",
        "            x = conv_layer(x)\n",
        "\n",
        "            if self.use_batch_norm:\n",
        "                x = BatchNormalization()(x)\n",
        "\n",
        "            x = LeakyReLU()(x)\n",
        "\n",
        "            if self.use_dropout:\n",
        "                x = Dropout(rate = 0.25)(x)\n",
        "\n",
        "        shape_before_flattening = K.int_shape(x)[1:]\n",
        "\n",
        "        x = Flatten()(x)\n",
        "        self.mu = Dense(self.z_dim, name='mu')(x)\n",
        "        self.log_var = Dense(self.z_dim, name='log_var')(x)\n",
        "\n",
        "        self.encoder_mu_log_var = Model(encoder_input, (self.mu, self.log_var))\n",
        "\n",
        "        def sampling(args):\n",
        "            mu, log_var = args\n",
        "            epsilon = K.random_normal(shape=K.shape(mu), mean=0., stddev=1.)\n",
        "            return mu + K.exp(log_var / 2) * epsilon\n",
        "\n",
        "        encoder_output = Lambda(sampling, name='encoder_output')([self.mu, self.log_var])\n",
        "\n",
        "        self.encoder = Model(encoder_input, encoder_output)\n",
        "        \n",
        "        \n",
        "\n",
        "        ### THE DECODER\n",
        "\n",
        "        decoder_input = Input(shape=(self.z_dim,), name='decoder_input')\n",
        "\n",
        "        x = Dense(np.prod(shape_before_flattening))(decoder_input)\n",
        "        x = Reshape(shape_before_flattening)(x)\n",
        "\n",
        "        for i in range(self.n_layers_decoder):\n",
        "            conv_t_layer = Conv2DTranspose(\n",
        "                filters = self.decoder_conv_t_filters[i]\n",
        "                , kernel_size = self.decoder_conv_t_kernel_size[i]\n",
        "                , strides = self.decoder_conv_t_strides[i]\n",
        "                , padding = 'same'\n",
        "                , name = 'decoder_conv_t_' + str(i)\n",
        "                )\n",
        "\n",
        "            x = conv_t_layer(x)\n",
        "\n",
        "            if i < self.n_layers_decoder - 1:\n",
        "                if self.use_batch_norm:\n",
        "                    x = BatchNormalization()(x)\n",
        "                x = LeakyReLU()(x)\n",
        "                if self.use_dropout:\n",
        "                    x = Dropout(rate = 0.25)(x)\n",
        "            else:\n",
        "                x = Activation('sigmoid')(x)\n",
        "\n",
        "            \n",
        "\n",
        "        decoder_output = x\n",
        "\n",
        "        self.decoder = Model(decoder_input, decoder_output)\n",
        "\n",
        "        ### THE FULL VAE\n",
        "        model_input = encoder_input\n",
        "        model_output = self.decoder(encoder_output)\n",
        "\n",
        "        self.model = Model(model_input, model_output)\n",
        "\n",
        "\n",
        "    def compile(self, learning_rate, r_loss_factor):\n",
        "        self.learning_rate = learning_rate\n",
        "\n",
        "        ### COMPILATION\n",
        "        def vae_r_loss(y_true, y_pred):\n",
        "            r_loss = K.mean(K.square(y_true - y_pred), axis = [1,2,3])\n",
        "            return r_loss_factor * r_loss\n",
        "\n",
        "        def vae_kl_loss(y_true, y_pred):\n",
        "            kl_loss =  -0.5 * K.sum(1 + self.log_var - K.square(self.mu) - K.exp(self.log_var), axis = 1)\n",
        "            return kl_loss\n",
        "\n",
        "        def vae_loss(y_true, y_pred):\n",
        "            r_loss = vae_r_loss(y_true, y_pred)\n",
        "            kl_loss = vae_kl_loss(y_true, y_pred)\n",
        "            return  r_loss + kl_loss\n",
        "\n",
        "        optimizer = Adam(lr=learning_rate)\n",
        "        self.model.compile(optimizer=optimizer, loss = vae_loss,  metrics = [vae_r_loss, vae_kl_loss])\n",
        "\n",
        "\n",
        "    def train(self, x_train, batch_size, epochs, run_folder, print_every_n_batches = 100, initial_epoch = 0, lr_decay = 1):\n",
        "\n",
        "        lr_sched = ReduceLROnPlateau(monitor='loss', factor=0.01, patience=2, verbose=1)\n",
        "        \n",
        "        callbacks_list = [lr_sched]\n",
        "\n",
        "        self.model.fit(     \n",
        "            x_train\n",
        "            , x_train\n",
        "            , batch_size = batch_size\n",
        "            , shuffle = True\n",
        "            , epochs = epochs\n",
        "            , initial_epoch = initial_epoch\n",
        "            , callbacks = callbacks_list\n",
        "        )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0P9D_Ori4PXe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vae = VariationalAutoencoder(\n",
        "    input_dim = (28,28,1)\n",
        "    , encoder_conv_filters = [32,64,64, 64]\n",
        "    , encoder_conv_kernel_size = [3,3,3,3]\n",
        "    , encoder_conv_strides = [1,2,2,1]\n",
        "    , decoder_conv_t_filters = [64,64,32,1]\n",
        "    , decoder_conv_t_kernel_size = [3,3,3,3]\n",
        "    , decoder_conv_t_strides = [1,2,2,1]\n",
        "    , z_dim = 2\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PSHsDonH4fxo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "LEARNING_RATE = 0.0005\n",
        "R_LOSS_FACTOR = 1000\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}