BERT_COLING_1506.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_COLING_1506.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "D8QPOnRkPVpw",
        "UfeDhiIBZlZO",
        "UxJ1ZFI00jHU",
        "strutgjWHx78",
        "DsYo9XkFls7A",
        "q8VOIC_9H_oJ",
        "LoO2dIhRVyKP",
        "iGe2-2vBV-bE",
        "QFvhsyGNVzG9",
        "JzQjBPENWGJD",
        "Jge74Vq-WRvM",
        "d9vdM7GMoCVy",
        "9KNNF1iFTChI"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/esohman/sentimentator/blob/master/BERT_COLING_1506.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E9nfvbvkS9zZ",
        "colab_type": "text"
      },
      "source": [
        "# BERT for emotion detection"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D8QPOnRkPVpw",
        "colab_type": "text"
      },
      "source": [
        "## Files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GTeZGhyf2Mlq",
        "colab_type": "code",
        "outputId": "f919427b-bc22-49f4-a94f-8b1f400a1722",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 41
        }
      },
      "source": [
        "LOAD_FROM_LOCAL = True\n",
        "\n",
        "BASE_URL   = 'https://lthelsinkioffens.slack.com/files/UTPAU0RGF/F015V2CCFPA/'\n",
        "TRAIN_PATH = 'adx.train'\n",
        "DEV_PATH   = 'adx.dev'\n",
        "TEST_PATH  = 'adx.test'\n",
        "\n",
        "if LOAD_FROM_LOCAL:\n",
        "  from google.colab import files\n",
        "  uploaded = files.upload()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-0ce3eb32-fbe6-435f-8aba-150f680408b3\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-0ce3eb32-fbe6-435f-8aba-150f680408b3\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XAOWFGhZKLCh",
        "colab_type": "text"
      },
      "source": [
        "## Settings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NnJ9u9bTOtph",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Hyperparameters\n",
        "\n",
        "SEED = 12345 #@param {type:\"raw\"}\n",
        "\n",
        "BERT_MODEL = 'english_base_uncased' #@param [\"multilingual\", \"english_base_cased\", \"english_large_cased\", \"english_base_uncased\", \"english_large_uncased\", \"finnish_cased\", \"finnish_uncased\", \"dutch\", \"chinese\", \"german\", \"arabic\", \"greek\", \"turkish\"]\n",
        "#DO_PREPROCESSING = False #@param {type:\"boolean\"}\n",
        "#DO_BALANCING = False #@param {type:\"boolean\"}\n",
        "\n",
        "EPOCHS = 3 #@param [\"2\", \"3\", \"4\"] {type:\"raw\"}\n",
        "MAX_LEN =  48  #@param [\"32\", \"48\", \"64\", \"128\", \"256\", \"512\"] {type:\"raw\"}\n",
        "BATCH_SIZE = 96 #@param [\"128\", \"96\", \"64\", \"32\", \"16\", \"8\"] {type:\"raw\"}\n",
        "LEARN_RATE = 2e-5 #@param [\"3e-4\", \"1e-4\", \"5e-5\", \"3e-5\", \"2e-5\"] {type:\"raw\"}\n",
        "EPSILON = 1e-8 #@param [\"1e-6\", \"1e-7\", \"1e-8\"] {type:\"raw\"}\n",
        "nb_warmup_steps = 0 #@param {type:\"raw\"}\n",
        "\n",
        "#@markdown ---\n",
        "CROSS_VALIDATION = False #@param {type:\"boolean\"}\n",
        "NUM_FOLDS  = 5 #@param [\"3\", \"5\", \"10\"] {type:\"raw\"}\n",
        "\n",
        "CHANGE_SPLIT = True #@param {type:\"boolean\"}\n",
        "PCTG_TRAIN = 0.75 #@param {type:\"slider\", min:0, max:1, step:0.05}\n",
        "PCTG_DEV   = 0.15 #@param {type:\"slider\", min:0, max:1, step:0.05}\n",
        "PCTG_TEST  = 0.1 #@param {type:\"slider\", min:0, max:1, step:0.05}\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UfeDhiIBZlZO",
        "colab_type": "text"
      },
      "source": [
        "## Installs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Lk56ljMZkvr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install transformers"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UxJ1ZFI00jHU",
        "colab_type": "text"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tW3nTJKQ8PK3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys, io, os, re, csv, json, string, time, datetime, random, unicodedata, itertools, collections, torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from transformers import BertTokenizer, BertForSequenceClassification, AdamW, BertConfig, get_linear_schedule_with_warmup\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler, random_split\n",
        "from sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n",
        "from sklearn.metrics import classification_report, confusion_matrix, f1_score, precision_score, recall_score, accuracy_score\n",
        "from sklearn.utils import resample\n",
        "\n",
        "% matplotlib inline\n",
        "random.seed(SEED)\n",
        "np.random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "torch.cuda.manual_seed_all(SEED)\n",
        "pd.set_option('precision', 4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "strutgjWHx78",
        "colab_type": "text"
      },
      "source": [
        "## Dictionaries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oXjt0mN5Hyas",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "models = {\n",
        "    \"english_base_cased\": ('bert-base-cased',False),\n",
        "    \"english_large_cased\": ('bert-large-cased',False),\n",
        "    \"english_base_uncased\": ('bert-base-uncased',True),\n",
        "    \"english_large_uncased\": ('bert-large-uncased',True),\n",
        "    \"multilingual\": ('bert-base-multilingual-cased',False),\n",
        "    \"finnish_cased\": ('TurkuNLP/bert-base-finnish-cased-v1',False),\n",
        "    \"finnish_uncased\": ('TurkuNLP/bert-base-finnish-uncased-v1',True),\n",
        "    \"dutch\": ('wietsedv/bert-base-dutch-cased',False),\n",
        "    \"chinese\": ('bert-base-chinese',False),\n",
        "    \"german\": ('bert-base-german-cased',False),\n",
        "    \"arabic\": ('asafaya/bert-base-arabic',False),\n",
        "    \"greek\": ('nlpaueb/bert-base-greek-uncased-v1',True),\n",
        "    \"turkish\": ('dbmdz/bert-base-turkish-cased',False)\n",
        "}\n",
        "\n",
        "datasets_olid = {\n",
        "    \"arabic\": \"https://github.com/mapama247/TFM/raw/master/ar20a.tsv\",\n",
        "    \"danish\":  \"https://github.com/mapama247/TFM/raw/master/da20a.tsv\",\n",
        "    \"english\": \"https://github.com/mapama247/TFM/raw/master/en19a.tsv\",\n",
        "    \"greek\":   \"https://github.com/mapama247/TFM/raw/master/gr20a.tsv\",\n",
        "    \"turkish\": \"https://github.com/mapama247/TFM/raw/master/tr20a.tsv\"\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DsYo9XkFls7A",
        "colab_type": "text"
      },
      "source": [
        "## Environment setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eXaJrfBOJ0EM",
        "colab_type": "code",
        "outputId": "caa22a7d-5e34-4e1f-e8ea-15e1588d3ac4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "if tf.test.gpu_device_name() == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(tf.test.gpu_device_name()))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda\")\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('Using', torch.cuda.get_device_name(0), ':)')\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n",
            "There are 1 GPU(s) available.\n",
            "Using Tesla T4 :)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q8VOIC_9H_oJ",
        "colab_type": "text"
      },
      "source": [
        "## Aux functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LoO2dIhRVyKP",
        "colab_type": "text"
      },
      "source": [
        "### Formatting data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VEt3WTVSVyns",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def prepare_data(sentences, labels, random_sampling=False):\n",
        "  input_ids=[]\n",
        "  attention_masks = []\n",
        "  for sentence in sentences:\n",
        "    encoded_dict = tokenizer.encode_plus(sentence, add_special_tokens=True, max_length=MAX_LEN, pad_to_max_length=True, return_attention_mask=True, return_tensors='pt')\n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "  \n",
        "  # Convert numpy arrays to pytorch tensors\n",
        "  inputs = torch.cat(input_ids, dim=0)\n",
        "  masks =  torch.cat(attention_masks, dim=0)\n",
        "  labels = torch.tensor(labels)\n",
        "\n",
        "  # Create the DataLoader for the given set (iterator to save memory)\n",
        "  data = TensorDataset(inputs, masks, labels)\n",
        "  if random_sampling: # train data\n",
        "      sampler = RandomSampler(data)\n",
        "  else: # dev and test data\n",
        "      sampler = SequentialSampler(data)\n",
        "  dataloader = DataLoader(data, sampler=sampler, batch_size=BATCH_SIZE)\n",
        "  \n",
        "  return dataloader"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iGe2-2vBV-bE",
        "colab_type": "text"
      },
      "source": [
        "### Train model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z3S6MH17V-7n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(model, optimizer, scheduler, train_dataloader, dev_dataloader, epochs, verbose=True):\n",
        "  total_t0 = time.time() # Measure the total training time for the whole run.\n",
        "\n",
        "  training_stats = [] # training loss, validation loss, validation accuracy and timings.\n",
        "\n",
        "  for epoch_i in range(0, epochs):\n",
        "      if verbose: print('\\n======== Epoch {:} / {:} ========'.format(epoch_i+1, epochs))\n",
        "      else: print(\"Epoch\",epoch_i+1,\"of\",epochs,\"...\")\n",
        "      \n",
        "      t0 = time.time()     # Measure how long the training epoch takes.\n",
        "      total_train_loss = 0 # Reset the total loss for this epoch.\n",
        "      model.train()        # Put the model into training mode.\n",
        "\n",
        "      # For each batch of training data...\n",
        "      for step, batch in enumerate(train_dataloader):\n",
        "          \n",
        "          if verbose==True and step%40==0 and not step==0: # Progress update every 40 batches.\n",
        "              elapsed = format_time(time.time()-t0) # Calculate elapsed time in minutes.\n",
        "              print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "          # Unpack this training batch from our dataloader and copy each tensor to the GPU using the to() method.\n",
        "          b_input_ids = batch[0].to(device)\n",
        "          b_input_mask = batch[1].to(device)\n",
        "          b_labels = batch[2].to(device)\n",
        "\n",
        "          # Always clear any previously calculated gradients before performing a backward pass.\n",
        "          model.zero_grad()\n",
        "\n",
        "          # Perform a forward pass (evaluate the model on this training batch).\n",
        "          loss, logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "\n",
        "          # Accumulate the training loss over all of the batches so that we can calculate the average loss at the end.\n",
        "          total_train_loss += loss.item()\n",
        "\n",
        "          # Perform a backward pass to calculate the gradients.\n",
        "          loss.backward()\n",
        "\n",
        "          # Clip the norm of the gradients to 1.0 in order to prevent the \"exploding gradients\" problem.\n",
        "          torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "          optimizer.step() # Update parameters and take a step using the computed gradient.\n",
        "          scheduler.step() # Update the learning rate.\n",
        "\n",
        "      avg_train_loss = total_train_loss / len(train_dataloader) # Calculate the average loss over the training data.\n",
        "      if verbose: print(\"\\n  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "      \n",
        "      # Measure how long this epoch took.\n",
        "      training_time = format_time(time.time() - t0)\n",
        "      if verbose: print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "\n",
        "      ########## VALIDATION ##########\n",
        "      t0 = time.time()\n",
        "      model.eval() # Put the model in evaluation mode--the dropout layers behave differently during evaluation.\n",
        "      eval_loss, eval_accuracy, nb_eval_examples = 0, 0, 0      # Tracking variables \n",
        "\n",
        "      # Tracking variables \n",
        "      total_eval_accuracy, total_eval_loss, nb_eval_steps = 0, 0, 0\n",
        "\n",
        "      # Evaluate data for one epoch\n",
        "      for batch in dev_dataloader:\n",
        "          # Unpack the inputs from the dataloader object after adding batch to GPU\n",
        "          b_input_ids = batch[0].to(device)\n",
        "          b_input_mask = batch[1].to(device)\n",
        "          b_labels = batch[2].to(device)\n",
        "          \n",
        "          # Telling the model not to compute or store gradients, saving memory and speeding up validation\n",
        "          with torch.no_grad():        \n",
        "              # Forward pass, calculate logit predictions (output values prior to applying an activation function)\n",
        "              (loss, logits) = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "\n",
        "          logits = logits.detach().cpu().numpy() # Move logits to CPU\n",
        "          label_ids = b_labels.to('cpu').numpy() # Move labels to CPU\n",
        "\n",
        "          total_eval_loss += loss.item() # Accumulate the validation loss.\n",
        "          total_eval_accuracy += flat_accuracy(logits, label_ids) # Calculate the accuracy for this batch of test sentences, and accumulate it over all batches.\n",
        "\n",
        "          nb_eval_steps += 1 # Track the number of batches\n",
        "\n",
        "      # Report the final accuracy for this validation run.\n",
        "      avg_val_accuracy = total_eval_accuracy / len(dev_dataloader)\n",
        "      if verbose: print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "      # Report the average loss over all of the batches.\n",
        "      avg_val_loss = total_eval_loss / len(dev_dataloader)\n",
        "      if verbose: print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "\n",
        "      # Measure how long the validation run took.\n",
        "      validation_time = format_time(time.time() - t0)\n",
        "      if verbose: print(\"  Validation took: {:}\".format(validation_time))  \n",
        "\n",
        "      # Record all statistics from this epoch.\n",
        "      training_stats.append(\n",
        "          {\n",
        "              'epoch': epoch_i + 1,\n",
        "              'Train Loss': avg_train_loss,\n",
        "              'Valid Loss': avg_val_loss,\n",
        "              'Valid Acc': avg_val_accuracy,\n",
        "              'Train Time': training_time,\n",
        "              'Valid Time': validation_time\n",
        "          }\n",
        "      )\n",
        "\n",
        "  print(\"\\nTraining complete!\")\n",
        "  print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))\n",
        "\n",
        "  df_stats = pd.DataFrame(data=training_stats)\n",
        "  df_stats = df_stats.set_index('epoch')\n",
        "  return df_stats"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XOZpAMdUWC20",
        "colab_type": "text"
      },
      "source": [
        "### Make predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gI8DZPx4WDNp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predict(model, dataloader):\n",
        "  print('Predicting labels for test sentences...')\n",
        "\n",
        "  model.eval() # Put model in evaluation mode\n",
        "\n",
        "  predictions = [] # Tracking variable\n",
        "  true_labels = [] # Tracking variable\n",
        "\n",
        "  for batch in dataloader:\n",
        "    batch = tuple(t.to(device) for t in batch) # Add batch to GPU\n",
        "    \n",
        "    b_input_ids, b_input_mask, b_labels = batch # Unpack the inputs from the dataloader\n",
        "    \n",
        "    with torch.no_grad(): # do not compute or store gradients to save memory and speed up prediction\n",
        "      outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask) # Forward pass, calculate logit predictions\n",
        "    \n",
        "    logits = outputs[0] # retrieve the model outputs prior to activation\n",
        "    logits = logits.detach().cpu().numpy() # Move logits to CPU\n",
        "    label_ids = b_labels.to('cpu').numpy() # Move labels to CPU\n",
        "    \n",
        "    predictions.append(logits)    # Store predictions\n",
        "    true_labels.append(label_ids) # Store true labels\n",
        "\n",
        "  print('COMPLETED.\\n')\n",
        "  return predictions, true_labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QFvhsyGNVzG9",
        "colab_type": "text"
      },
      "source": [
        "### Evaluate model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b8QVvvNnWGtI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(predictions, true_labels, avg='macro', verbose=True):\n",
        "  avgs = ['micro', 'macro', 'weighted', 'samples']\n",
        "  if avg not in avgs:\n",
        "    raise ValueError(\"Invalid average type (avg). Expected one of: %s\" % avgs)\n",
        "\n",
        "  # Combine the predictions for each batch into a single list.\n",
        "  flat_predictions = [item for sublist in predictions for item in sublist]\n",
        "  flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
        "\n",
        "  # Combine the correct labels for each batch into a single list.\n",
        "  flat_true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Compute the results.\n",
        "  precision = precision_score(flat_true_labels, flat_predictions, average=avg)\n",
        "  recall    = recall_score(flat_true_labels, flat_predictions, average=avg)\n",
        "  f1        = f1_score(flat_true_labels, flat_predictions, average=avg)\n",
        "  acc       = accuracy_score(flat_true_labels,flat_predictions)\n",
        "\n",
        "  # Report the results.\n",
        "  if verbose:\n",
        "    print('Accuracy:        %.4f' % acc)\n",
        "    print(avg+' Precision: %.4f' % f1)\n",
        "    print(avg+' Recall:    %.4f' % f1)\n",
        "    print(avg+' F1 score:  %.4f' % f1, \"\\n\")\n",
        "    print(confusion_matrix(flat_true_labels,flat_predictions))\n",
        "    #print(classification_report(flat_true_labels, flat_predictions, digits=2, zero_division='warn'))\n",
        "\n",
        "  return f1, acc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JzQjBPENWGJD",
        "colab_type": "text"
      },
      "source": [
        "### Plots"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hcJQkd4CWF3U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_lengths_distribution(lengths, title=\"Length distribution of tokenized sentences\"):\n",
        "  sns.set(style=\"darkgrid\")\n",
        "  sns.set(font_scale=1.5)\n",
        "  plt.rcParams[\"figure.figsize\"]=(10,5)\n",
        "  lengths = [min(length,MAX_LEN) for length in lengths]\n",
        "  ax = sns.distplot(lengths,kde=False,rug=False, hist_kws={\"rwidth\":5,'edgecolor':'black', 'alpha':1.0}) \n",
        "  plt.title(\"Sequence length distribution\")\n",
        "  plt.xlabel(\"Sequence Length\")\n",
        "  plt.ylabel(\"Counts\")\n",
        "\n",
        "  num_truncated = lengths.count(MAX_LEN)\n",
        "  num_sentences = len(lengths)\n",
        "  print(\"{:.1%} of the training examples ({:,} of them) have more than {:,} tokens\".format(float(num_truncated)/float(num_sentences),num_truncated,MAX_LEN))\n",
        "\n",
        "def plot_value_counts(df, title=\"Label distribution\"):\n",
        "  emotions = ['TRUST','ANGRY','ANTICIP.','DISGUST','FEAR','JOY','SADNESS','SURPRISE']\n",
        "  df2 = df.replace([0,1,2,3,4,5,6,7], emotions)\n",
        "  df2.label.value_counts(normalize=False).sort_index().plot(kind='bar')\n",
        "  plt.xticks(rotation=25)\n",
        "  plt.title(title)\n",
        "  plt.show()\n",
        "\n",
        "def plot_loss(df_stats, plot_train=True, plot_valid=True):\n",
        "  sns.set(style='darkgrid') # Use plot styling from seaborn.\n",
        "  sns.set(font_scale=1.5) # Increase the plot size.\n",
        "  plt.rcParams[\"figure.figsize\"] = (12,6) # Increase the font size.\n",
        "  if plot_train: plt.plot(df_stats['Train Loss'], 'b-o', label=\"Training\")\n",
        "  if plot_valid: plt.plot(df_stats['Valid Loss'], 'g-o', label=\"Validation\")\n",
        "  plt.title(\"Training & Validation Loss\")\n",
        "  plt.xlabel(\"Epoch\")\n",
        "  plt.ylabel(\"Loss\")\n",
        "  plt.legend()\n",
        "  plt.xticks([1, 2, 3, 4])\n",
        "  plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jge74Vq-WRvM",
        "colab_type": "text"
      },
      "source": [
        "### Aux"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9dfZV-djIAFO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def flat_accuracy(preds, labels):\n",
        "    \"\"\"Function to calculate the accuracy of our predictions vs labels\"\"\"\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
        "\n",
        "def format_time(elapsed):\n",
        "    \"\"\"Function for formatting elapsed times: Takes a time in seconds and returns a string hh:mm:ss\"\"\"\n",
        "    elapsed_rounded = int(round((elapsed))) # Round to the nearest second.\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded)) # Format as hh:mm:ss\n",
        "\n",
        "def print_model_params(my_model):\n",
        "  \"\"\"Function to print all the model's parameters as a list of tuples: (name,dimensions)\"\"\"\n",
        "  params = list(my_model.named_parameters())\n",
        "  print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "  print('==== Embedding Layer ====\\n')\n",
        "  for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "  print('\\n==== First Transformer ====\\n')\n",
        "  for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "  print('\\n==== Output Layer ====\\n')\n",
        "  for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2XkARNl3lyDI",
        "colab_type": "text"
      },
      "source": [
        "## Read data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ps76HJlq-OZD",
        "colab_type": "code",
        "outputId": "55bce020-c206-4606-cfa4-5196fe65c19b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 526
        }
      },
      "source": [
        "df_train = pd.read_csv(\"adx_train.txt\", delimiter='\\t', header=None, names=['label','sentence']).sample(frac=1, random_state=SEED)\n",
        "df_dev = pd.read_csv(\"adx_dev.txt\"  , delimiter='\\t', header=None, names=['label','sentence']).sample(frac=1, random_state=SEED)\n",
        "df_test  = pd.read_csv(\"adx_test.txt\" , delimiter='\\t', header=None, names=['label','sentence']).sample(frac=1, random_state=SEED)\n",
        "\n",
        "df_train[\"label\"].replace({8: 0}, inplace=True)\n",
        "df_dev[\"label\"].replace({8: 0}, inplace=True)\n",
        "df_test[\"label\"].replace({8: 0}, inplace=True)\n",
        "\n",
        "df_data = pd.concat([df_train, df_dev, df_test]).sample(frac=1, random_state=SEED)\n",
        "NUM_CLASSES = len(df_data.groupby('label'))\n",
        "\n",
        "if CHANGE_SPLIT:\n",
        "  if PCTG_TRAIN+PCTG_DEV+PCTG_TEST!=1:\n",
        "    raise SystemError('Check train-dev-test percentages! The sum of them should be 1.')\n",
        "  else:\n",
        "    print('The entire dataset was split using',PCTG_TRAIN,'for training,',PCTG_DEV,'for validation and',PCTG_TEST,'for testing.\\n')\n",
        "    df_train, df_dev, df_test = np.split(df_data, [int(PCTG_TRAIN*len(df_data)), int((1-PCTG_TEST)*len(df_data))])\n",
        "else:\n",
        "  print('The original train-dev-test sets are used.\\n')\n",
        "\n",
        "X_train = df_train.sentence.values\n",
        "y_train = df_train.label.values\n",
        "X_dev = df_dev.sentence.values\n",
        "y_dev = df_dev.label.values\n",
        "X_test = df_test.sentence.values\n",
        "y_test = df_test.label.values\n",
        "\n",
        "print('Number of sentences train set: {:,}'.format(len(X_train)))\n",
        "print('Number of sentences dev set:   {:,}'.format(len(X_dev)))\n",
        "print('Number of sentences test set:  {:,}'.format(len(X_test)))\n",
        "\n",
        "plot_value_counts(df_data, title=\"Value counts labels full dataset\")\n",
        "#plot_value_counts(df_train, title=\"Value counts labels train set\")\n",
        "#plot_value_counts(df_dev, title=\"Value counts labels dev set\")\n",
        "#plot_value_counts(df_test, title=\"Value counts labels test set\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The entire dataset was split using 0.75 for training, 0.15 for validation and 0.1 for testing.\n",
            "\n",
            "Number of sentences train set: 18,123\n",
            "Number of sentences dev set:   3,624\n",
            "Number of sentences test set:  2,417\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt8AAAGkCAYAAADpB0lbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzde1yUdf7//6coBwVMdEctz6Yggajgx0T9WCab6HrAAg8pymqWrbdK7bOJHb77+VSmIZ7KyrRijdU8BU1lmmltretp1+Oqk6ZrpRkyeUBRYUDm94c/rm0aBMbGC8jH/XbzVryv1/We63rNlE8v39c1tZxOp1MAAAAAbjifqj4AAAAA4GZB+AYAAABMQvgGAAAATEL4BgAAAExC+AYAAABMQvgGAAAATEL4BuAVJ06cUFhYmF555ZWqPhRICgsLU2pq6nXvf8899yg5OdmLR3TVjficlJSU6JVXXlHfvn11xx13KCwszOM5tm/frrCwMGVlZXn9WG9ULwHUTIRv4Cby2GOPKSwsTDab7Zo1TqdT99xzj7p27aqCggITj+7mc+LECb3yyivlvh+oWHZ2thYuXKg777xTM2bMUFpaWlUfklf9+c9/dvlDQXXB5xe4PoRv4CaSmJgoSXrvvfeuWbNt2zZ9//33GjBggAICAsw6tJvS999/r4ULFxJefqEtW7YoODhYM2bM0NChQzVkyJCqPiSveuedd5SdnV3Vh+GGzy9wfQjfwE2kV69euvXWW/Xhhx/K4XCUWVN6ha00qAPVnd1uV/369VWrVq2qPhQAqBDhG7iJ+Pj4aOjQoTp37pw+++wzt+35+fnasGGDQkNDFRUVpfz8fM2bN09JSUm68847FRkZqd/+9rdKT0/X5cuXK3y9stbRlkpNTS1zbe4333yjP/7xj+rVq5ciIyN1zz336KWXXtKlS5cqfZ4HDx7UY489ph49eigyMlJ33XWXpk6dqu+++86lbvXq1Ro6dKiioqIUExOjcePG6Z///KdLTXnrfl955RWFhYXpxIkTbud14cIF/elPf1JsbKw6duyoESNGaO/evUZdVlaWxowZI0maPn26wsLCFBYWZqwNLikp0Z///GcNGjRIXbp0UXR0tPr166ennnpKRUVFle7FT3388ceaOHGi7r77bkVGRurOO+/UH/7wB3311VfX3OfAgQMaM2aMunTpom7dumnatGk6ffq0W53D4dCiRYv0u9/9Th07dlTXrl01ceJEHTx4sFLH9v777ysxMVFdu3ZV586d1bdvXz3xxBM6c+bMNfcp/Xxt375d33//vdHD0rXu11prXd7n8pf44Ycf9PjjjysmJkbR0dGaOHGi22euVGXfi7CwMH3//ffasWOHcX4//cxt3rxZkydPVt++fRUVFaWuXbtq3Lhx2rFjh9trfv3113rsscf03//934qMjFTPnj2VnJysv/71ry51lXkvK/r8Ari2OlV9AADMdd999+n1119XVlaW4uPjXbatXbtWBQUFuv/++yVJp06d0po1a3Tvvfdq4MCBqlOnjnbs2KE333xTNptNb731llePbf/+/Ro7dqzq16+v4cOHq0mTJvrqq6+UmZmp3bt3KzMzU76+vuXO8fnnn+vRRx9VvXr1lJiYqFatWslut2vz5s06fPiwWrZsKUmaPXu23nzzTUVFRWnq1KnKz8/XqlWrNHbsWL322mu66667ftG5jB8/Xg0bNtSkSZN07tw5ZWRk6KGHHtKmTZsUFBSk//qv/9LEiRO1aNEiDR8+XDExMZKk3/zmN5Kk119/XS+//LL69OmjESNGqHbt2jpx4oQ+++wzORyOCvtQlr/85S9q0KCBhg0bJovFou+++06rVq3SyJEjlZ2drdatW7vU5+TkKCUlRffee6/69eungwcP6r333tP+/fu1Zs0a1a1bV5JUVFSk8ePHa/fu3RoyZIhGjRpl9HPkyJH6y1/+oo4dO17zuN5//31NmzZNXbt21WOPPaaAgAD98MMP+uKLL3T69Gk1bNiwzP1uv/12paWladGiRTp79qymT58uScZ7bKbz589r1KhRysnJ0YgRI3T77bfrH//4h8aMGVPmvROVfS/S0tI0c+ZMhYSEaOLEicb+pT3Jzs5WXl6eEhIS1LRpU506dUqrV69WSkqK3nnnHXXt2lWSdPbsWY0dO1aSNGLECN122206e/as9u/fr7179+ruu++WVPn3sqLPL4ByOAHcdMaMGeMMDw93njp1ymV82LBhzoiICOfp06edTqfTWVhY6HQ4HG77z5s3zxkaGurcu3evMXb8+HFnaGio8+WXXzbGtm3b5gwNDXW+9957bnNMmzbNGRoa6jI2aNAgZ79+/ZwXLlxwGd+wYcM15/mpS5cuOe+8805n9+7dnTk5OW7br1y54nQ6nc6jR486w8LCnCNGjHAWFhYa23NycpwxMTHOPn36OIuLi695XqVefvllZ2hoqPP48eNu5/WnP/3Jpfbjjz92hoaGOt99911jrLz+JCQkOPv371/u+ZYnNDTUOW3aNJexixcvutUdOXLEGRER4Xa8ffr0cYaGhjozMjJcxjMyMpyhoaHON954w23syy+/dKm9cOGC86677nKOHj3aGCurn5MmTXJ26dLFWVRU5OlpOp1Op3P06NHOPn36uI336dPH5bVLldX3ssbKe+9/bs6cOc7Q0FDnmjVrXMZfeOEFZ2hoqNtxePpelHUe15rHbrc7u3Xr5nzwwQeNsY0bNzpDQ0Oda9euLfc8PHkvy/v8Arg2lp0AN6HExERduXJF77//vjF29OhR7dmzR/fcc49xVc3Pz8+4wlpcXKy8vDydOXNGPXr0kCSXZRS/1KFDh3To0CENHDhQDodDZ86cMX7FxMSoXr16+vvf/17uHJs3b9bZs2f1+9//Xk2aNHHb7uNz9X95mzZtktPp1IMPPig/Pz9je5MmTXTffffp+++/r/RyiWtJSUlx+bl79+6SpG+//bZS+wcFBenUqVNuy2B+iXr16km6+kSb/Px8nTlzRiEhIWrTpo327dtX5jE88MADLmMPPPCAgoKC9OmnnxpjH3zwgdq2bauIiAiX983hcKhHjx7auXNnuU/OCQ4OVkFBgf7617/K6XR66WzNtXHjRv3mN79RQkKCy/iECRPKrPf0vbiW0nkk6eLFizp79qx8fHzUqVMnl3mCg4MlSX/729+Un59/zfl+6XsJoGIsOwFuQvfee6/q16+vrKwsPfTQQ5L+8wSU0iUnpZYtW6YVK1boyJEjKikpcdmWl5fntWM6evSopKvrqK/1XOUff/yx3Dm++eYbSdIdd9xRbl3petn27du7bSsdO378eLlLJSrSokULl59DQkIkSefOnavU/lOnTtWkSZM0atQoNW7cWN26ddPdd9+tfv36ufyBwRMHDx7UggULtGPHDrc19M2bNy/zHH7+Wn5+fmrRooWOHz9ujB09elQFBQWKjY295mufPXtWt956a5nbHn74Yf3jH//QpEmT1KBBA3Xr1k29e/dW//79FRQU5MkpVpnSz0vt2rVdxhs3bqz69eu71Xv6XlzLd999p3nz5mnz5s06f/68y7af3oDarVs3JSQkKCsrSx9++KEiIyPVo0cPDRgwQO3atTPqful7CaBihG/gJuTv76+BAwdq+fLl2rVrlzp16qQPPvhATZs21X//938bdRkZGZo1a5Z69eqlMWPGqHHjxvL19dWpU6eUmppa4VXK8p4+UVxcXOb4uHHjXI7hp8oKMTfa9ZyDJLcQVqqyV3a7dOmiTz/9VJs3b9b27du1fft2ffTRR3r99de1fPlyNWjQoFLzlDp58qRGjRqloKAgPfLII2rbtq3q1q2rWrVq6cUXX/TohtafczqdCg0NNdZcl+Va67YlqXXr1vr444+1detWbd26VTt27NAzzzyjl19+WcuWLfP6Gu4rV654dT5Peeu9uHjxokaNGqXLly9r7NixCg0NVWBgoHx8fPTGG29o27ZtLvUvvfSSxo8fry+//FL//Oc/lZGRoUWLFumpp57S6NGjJf3y9xJAxQjfwE0qMTFRy5cvV1ZWlvLy8mS32zVx4kRjaYYkWa1WNWvWTEuWLHEZ//LLLyv1Grfccouksq+Q//QJIZLUqlUrSVeXhpQua/FUmzZtJEk2m029evW6Zl3pVemvv/7aLdgdOXLEpcaTc/BURY/GCwwMVL9+/dSvXz9JV/8W4rnnntOaNWv04IMPevRan376qS5duqTXX3/dWAJT6ty5c2VeTT9+/LgcDofLNofDoePHj6tt27bGWKtWrXT27Fl1797d5XPiCT8/P911113Gja5ffPGFHnroIWVkZOhPf/rTdc3ZoEGDMv+m4adX7b2lRYsW+vbbb3XlyhWXP3jl5ua6XZG+nveiLFu3blVubq5efPFFt7+xmj9/fpn7hIaGKjQ0VA8++KDOnz+vpKQkzZkzR6NGjVKtWrU8ei95tCNwfVjzDdykIiIiFB4ero8//ljLli1TrVq13J7t7ePjo1q1arlcrS0uLtaSJUsq9RrNmzdXnTp1tGXLFpfxXbt2ac+ePS5jd9xxh0JDQ7VixYoyw1FxcXGFSzZ69uypkJAQZWRkKDc312176Xncc889qlWrlt566y2Xx/bl5uYqKytLzZo1M5auBAUFyWKxaNu2bS59OH78uDZu3FhBB8pXul63rGBf1iP2IiIirllfkdJA+PMr76tWrZLdbi9zn/z8fC1fvtxlbPny5crPz1dcXJwxlpCQILvdroyMjDLnqWi5UFnnWtr/X7K0qXXr1jp27JhOnTpljDkcDi1btuy657yWvn376scff3S5j0JSmf+tePpeBAYGlvnZv9Y8mzdvdrsf49y5c27LxurXr6/mzZvr8uXLKiwslOTZe1ne5xfAtXHlG7iJJSYm6vnnn9ff/vY3devWzW2dcnx8vObMmaMJEybot7/9rfLz8/XRRx+pTp3K/a8jMDBQQ4cO1erVqzV16lR169ZN3377rbKyshQWFubyTONatWopLS1NY8eO1eDBg3X//ferXbt2Kigo0LfffqtPP/1UU6dO1X333XfN16tbt65mzJihxx9/XIMGDTIeNXjmzBlt3rxZKSkpiouLU9u2bTV+/Hi9+eabGj16tPr376+LFy9q1apVunTpktLT012uXo4aNUrz58/Xgw8+qLi4OOXm5mrFihVq3769/vWvf3nY9f9o166dAgMDtXz5cgUEBKh+/fpq2LChYmNjNWDAAHXu3FlRUVFq3Lix7Ha7Vq1aJV9fX/3ud7/z+LV69+6tunXr6sknn9To0aNVv3597dq1S19++aVatmxZ5lKMli1b6tVXX9XXX3+tiIgIHThwQO+9957atm3r8jznMWPGaMuWLUpLS9O2bdvUvXt3BQUF6eTJk9q2bZv8/PyUmZl5zWMbP368goOD1bVrV9166606f/68srOzVatWrV/0bZWjRo3S2rVrlZKSohEjRqioqEhWq9V4RKI3Pfjgg/roo4/07LPP6sCBA2rXrp127NihPXv2GOv9S3n6XnTq1Elr1qzR/Pnzdfvtt8vHx0d9+vRRTEyMLBaLXnrpJX3//fdq2rSpbDabrFarQkNDdfjwYWOO999/X0uXLlVcXJxatWqlOnXq6B//+Ic2b96s/v37G99m68l7Wd7nF8C1Eb6Bm9igQYOUlpamwsJCt7+2lq6GIqfTqTVr1mjGjBmyWCzq37+/7r//fg0YMKBSrzF9+nQ5nU5t3LhRmzZtUkREhF5//XWtWrXK7QtFwsPDlZ2drTfeeEOfffaZVqxYocDAQDVr1kxDhw6t1G/qffv21fLly7Vo0SKtWbNGFy9e1G9+8xvFxMS4fKnPH//4R7Vq1UrLly/XnDlz5Ovrq06dOmnOnDnGs5FLTZgwQRcuXNAHH3ygHTt2qF27dpoxY4YOHDjwi8J3QECA5s2bp/nz5+vFF1+Uw+FQt27dFBsbq3HjxumLL75QZmamLly4oEaNGqlTp056+OGH1aFDB49fq2XLllqyZInmzp2rRYsWqXbt2oqOjlZmZqaef/55ff/99277NG3aVPPnz9dLL72ktWvXytfXV4MGDdK0adNcnrLh6+urN954Q8uXL5fVajVumG3cuLE6duyooUOHlntsI0eO1Lp167Ry5Url5eWpQYMGCg8P1zPPPOO2LMMTMTExmjVrlhYtWqTZs2ercePGGjlypCIjI92eRvNL3XLLLVq2bJlmzZplXP3u1q2b3nnnHbfX8vS9mDJlivLy8rR8+XKdP39eTqdTmzZtUvPmzfXmm29q9uzZ+stf/qLi4mJFRkZqyZIlWrNmjUv4vvPOO2Wz2fTXv/5VdrtdPj4+at68uaZNm2as95Y8ey/L+/wCuLZazpr6XCcAAACghmHNNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYJKb7jnfZ89eVElJ9X66YqNGQTp9Or+qD+NXgV56F/30LvrpPfTSu+ind9FP76kJvfTxqaWQkMBrbr/pwndJibPah29JNeIYawp66V3007vop/fQS++in95FP72npveSZScAAACASQjfAAAAgEkI3wAAAIBJCN8AAACASQjfAAAAgEkI3wAAAIBJCN8AAACASQjfAAAAgEkI3wAAAIBJCN8AAACASQjfAAAAgEkI3wAAAIBJ6lT1AdR0wfXrKsDf+220WIK9NldBYbEunL/stfkAAABwfQjfv1CAfx0NesJa1YdRrg/nDNGFqj4IAAAAsOwEAAAAMAvhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwyXWF7yVLligsLExDhgxx27Zr1y6NHDlSnTp1Us+ePfXCCy/o8mX3Z0w7HA7Nnj1bvXr1UlRUlIYNG6atW7eW+XqVnRMAAACozjwO33a7Xa+//rrq1avnts1msyklJUWFhYVKTU1VYmKiVq5cqSlTprjVpqamaunSpRo8eLCefvpp+fj4aMKECdq9e/d1zwkAAABUZx5/yc6cOXMUGRkpp9Op8+fPu2ybO3euGjRooMzMTAUGBkqSmjdvrmeeeUZbt25VbGysJGnfvn1au3atpk+frpSUFElSQkKCBg4cqPT0dC1btszjOQEAAIDqzqMr3/v27dMHH3yg6dOnu23Lz8/Xli1blJCQYIRkSRoyZIjq1aundevWGWPr16+Xr6+vkpKSjDF/f38lJiZq586dys3N9XhOAAAAoLqrdPh2Op16/vnnlZCQoPDwcLfthw4dUnFxsSIjI13G/fz8FB4eLpvNZozZbDa1adPGJVBLUlRUlJxOp1HryZwAAABAdVfp8P3+++/ryJEjmjx5cpnb7Xa7JMlisbhts1gsxtXs0trGjRuXWSfJqPVkTgAAAKC6q9Sa7/z8fM2ZM0cPPfRQmaFZkgoKCiRdvSr9c/7+/sb20lpfX98y6ySpsLDQ4zkrq1GjII/3+TWwWIKr+hCqzM187jcC/fQu+uk99NK76Kd30U/vqem9rFT4fv311+Xr66vf//7316wJCAiQdPURgj9XWFhobC+tLSoqKrNO+k8I92TOyjp9Ol8lJU6P97uWmvIBsNsvVPUhVAmLJfimPfcbgX56F/30HnrpXfTTu+in99SEXvr41Cr3Ym+F4Ts3N1dLly7V448/rh9//NEYLywsVFFRkU6cOKHg4GBjaUjpUpGf+vkyk2stGSndt7TWkzkBAACA6q7CNd+nT59WUVGR0tPT1bdvX+PX3r17dfToUfXt21dLlixRaGio6tSpo/3797vs73A4ZLPZXG7S7NChg44dO6aLFy+61O7du9fYLsmjOQEAAIDqrsIr382bN9err77qNj5//nxdunRJTz31lFq3bq3g4GDFxsbKarXq4YcfNp5kYrVadenSJcXHxxv7xsfH6+2339bq1auN53w7HA5lZWUpOjpaTZo0kSSP5gQAAACquwrDd3BwsOLi4tzGly5dqtq1a7tsmzJlikaMGKHk5GQlJSUpJydHGRkZ6t27t3r06GHUderUSfHx8UpPT5fdblfLli2VnZ2tkydPaubMmS6vU9k5AQAAgOrO46+XL09ERIQyMjLk5+enmTNnavXq1Ro2bJgWLFjgVpuWlqbk5GRZrVa98MILKi4u1uLFixUTE3PdcwIAAADVWS2n0+m9R3/UADfiaSeDnrB6bb4b4cM5Q6r9ncE3Sk24K7omoZ/eRT+9h156F/30LvrpPTWhlxU97cSrV74BAAAAXBvhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADBJnao+AKBUcP26CvD3/kfSYgn26nwFhcW6cP6yV+cEAAA3B8I3qo0A/zoa9IS1qg+jQh/OGaILVX0QAACgRmLZCQAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGCSCm+4/Ne//qVFixbp4MGDOn36tIKDg9WhQwdNmjRJ0dHRRl1ycrJ27Njhtv+AAQM0b948lzGHw6EFCxbIarXq/Pnz6tChg6ZMmaLY2Fi3/Xft2qXZs2fr4MGDCgoKUv/+/fXEE0+obt2613O+AAAAQJWpMHwfP35cV65cUVJSkiwWiy5cuKAPP/xQo0eP1pIlS9SzZ0+j9rbbbtPkyZNd9m/WrJnbnKmpqdqwYYPGjBmjVq1aKTs7WxMmTFBmZqa6dOli1NlsNqWkpKhdu3ZKTU1VTk6O3n77bZ04cUKLFi36JecNAAAAmK7C8D1gwAANGDDAZWzkyJGKi4vTO++84xK+69evryFDhpQ73759+7R27VpNnz5dKSkpkqSEhAQNHDhQ6enpWrZsmVE7d+5cNWjQQJmZmQoMDJQkNW/eXM8884y2bt1a5pVyAAAAoLq6rjXfdevWVcOGDXX+/Hm3bcXFxbp48eI1912/fr18fX2VlJRkjPn7+ysxMVE7d+5Ubm6uJCk/P19btmxRQkKCEbwlaciQIapXr57WrVt3PYcOAAAAVJlKh+/8/HydOXNG//73vzV37lwdPnzY7crz0aNH1blzZ0VHR6tXr15atGiRSkpKXGpsNpvatGnjEqglKSoqSk6nUzabTZJ06NAhFRcXKzIy0qXOz89P4eHhRh0AAABQU1T6Gy6feuopffLJJ5IkX19fjRgxQhMnTjS2t2jRQnfeeafCwsKUn5+vjz76SPPmzdPJkyf13HPPGXV2u11NmjRxm99isUiSceXbbre7jP+8ds+ePZU9dAAAAKBaqHT4njRpkoYPH66cnBxZrVY5HA4VFRXJz89PkvTiiy+61A8dOlSPP/64Vq1apZSUFLVt21aSVFBQIF9fX7f5/f39JUmFhYVGnSRj/p/Xlm73VKNGQde1X01nsQRX9SH8qtzM/byZz/1GoJ/eQy+9i356F/30nprey0qH77CwMIWFhUmSBg8erPvvv1/Tp0/Xyy+/fM19xo0bp/Xr12v79u1G+A4ICFBRUZFbbWnoLg3hAQEBkq4+lrCs2tLtnjp9Ol8lJc7r2rcsNeUDYLdfqOpDqFBN6aVUM/p5I1gswTftud8I9NN76KV30U/vop/eUxN66eNTq9yLvdd1w6Wvr6/69u2rDRs2lHsFumnTppKkvLw8Y8xisRhLS36qdJlJ48aNjbqfjv+8trQOAAAAqCmu+xsuCwoK5HQ6y32yyfHjxyVJDRs2NMY6dOigY8eOue23d+9eY7skhYaGqk6dOtq/f79LncPhkM1mU3h4+PUeOgAAAFAlKgzfZ86ccRvLz8/XJ598oltvvVWNGjVSfn6+2/KQK1eu6I033pCPj4/LU1Hi4+NVVFSk1atXG2MOh0NZWVmKjo42bsYMDg5WbGysrFarS1C3Wq26dOmS4uPjPT9bAAAAoApVuOZ78uTJ8vf3V5cuXWSxWPTDDz8oKytLOTk5mjt3riTpwIEDeuKJJzRw4EC1bNlSly5d0rp167R//35NmDBBLVq0MObr1KmT4uPjlZ6eLrvdrpYtWyo7O1snT57UzJkzXV57ypQpGjFihJKTk5WUlKScnBxlZGSod+/e6tGjh5dbAQAAANxYFYbvwYMHy2q1KjMzU+fPn1dwcLA6d+6stLQ0devWTdLVr5WPjo7Whg0b9OOPP8rHx0ft27fXrFmzNHToULc509LSNH/+fFmtVuXl5SksLEyLFy9WTEyMS11ERIQyMjKUnp6umTNnKigoSMOGDdPUqVO9dPoAAACAeSoM34mJiUpMTCy3pkWLFuU+9eTn/P39NW3aNE2bNq3C2q5du2rFihWVnhsAAACorq77hksAAAAAniF8AwAAACYhfAMAAAAmIXwDAAAAJiF8AwAAACYhfAMAAAAmIXwDAAAAJiF8AwAAACYhfAMAAAAmIXwDAAAAJiF8AwAAACYhfAMAAAAmIXwDAAAAJiF8AwAAACYhfAMAAAAmIXwDAAAAJiF8AwAAACYhfAMAAAAmIXwDAAAAJqlT1QcAALi5BNevqwB/7//2Y7EEe3W+gsJiXTh/2atzAgDhGwBgqgD/Ohr0hLWqD6NCH84ZogtVfRAAfnVYdgIAAACYhPANAAAAmITwDQAAAJikwvD9r3/9S5MmTVKfPn0UFRWlnj17avz48dq1a5db7a5duzRy5Eh16tRJPXv21AsvvKDLl91vVnE4HJo9e7Z69eqlqKgoDRs2TFu3bi3z9Ss7JwAAAFDdVRi+jx8/ritXrigpKUnPPvusxo8frzNnzmj06NH6+9//btTZbDalpKSosLBQqampSkxM1MqVKzVlyhS3OVNTU7V06VINHjxYTz/9tHx8fDRhwgTt3r3bpc6TOQEAAIDqrsKnnQwYMEADBgxwGRs5cqTi4uL0zjvvqGfPnpKkuXPnqkGDBsrMzFRgYKAkqXnz5nrmmWe0detWxcbGSpL27duntWvXavr06UpJSZEkJSQkaODAgUpPT9eyZcuM16nsnAAAAEBNcF1rvuvWrauGDRvq/PnzkqT8/Hxt2bJFCQkJRkiWpF1j+pEAACAASURBVCFDhqhevXpat26dMbZ+/Xr5+voqKSnJGPP391diYqJ27typ3Nxcj+cEAAAAaoJKh+/8/HydOXNG//73vzV37lwdPnzYuPJ86NAhFRcXKzIy0mUfPz8/hYeHy2azGWM2m01t2rRxCdSSFBUVJafTadR6MicAAABQE1T6S3aeeuopffLJJ5IkX19fjRgxQhMnTpQk2e12SZLFYnHbz2KxaM+ePcbPdrtdTZo0KbNOknHl25M5PdGoUdB17VfTefub3252N3M/b+ZzvxHoZ/V2M78/N/O53wj003tqei8rHb4nTZqk4cOHKycnR1arVQ6HQ0VFRfLz81NBQYGkq1elf87f39/YLkkFBQXy9fUts06SCgsLjbrKzumJ06fzVVLivK59y1JTPgB2e/X/nraa0kupZvTzRrBYgm/ac78RbtZ+8t969XezfjZvFPrpPTWhlz4+tcq92FvpZSdhYWHq2bOn7r//fr311ls6cOCApk+fLkkKCAiQdPURgj9XWFhobC+tLSoqKrNO+k8I92ROAAAAoCa4rhsufX191bdvX23YsEEFBQXG0pDSpSI/Zbfb1bhxY+Nni8ViLC35eZ0ko9aTOQEAAICa4Lq/4bKgoEBOp1MXL15UaGio6tSpo/3797vUOBwO2Ww2hYeHG2MdOnTQsWPHdPHiRZfavXv3GtsleTQnAAAAUBNUGL7PnDnjNpafn69PPvlEt956qxo1aqTg4GDFxsbKarW6hGqr1apLly4pPj7eGIuPj1dRUZFWr15tjDkcDmVlZSk6Otq4GdOTOQEAAICaoMIbLidPnix/f3916dJFFotFP/zwg7KyspSTk6O5c+cadVOmTNGIESOUnJyspKQk5eTkKCMjQ71791aPHj2Muk6dOik+Pl7p6emy2+1q2bKlsrOzdfLkSc2cOdPltSs7JwAAAFATVBi+Bw8eLKvVqszMTJ0/f17BwcHq3Lmz0tLS1K1bN6MuIiJCGRkZSk9P18yZMxUUFKRhw4Zp6tSpbnOmpaVp/vz5slqtysvLU1hYmBYvXqyYmBiXOk/mBAAAAKq7CsN3YmKiEhMTKzVZ165dtWLFigrr/P39NW3aNE2bNs1rcwIAAKB6Ca5fVwH+lX6ydaV4+3GlBYXFunD+slfnLI93uwEAAAD8/wL862jQE9aqPoxyfThniMx8cvh1P+0EAAAAgGcI3wAAAIBJCN8AAACASQjfAAAAgEkI3wAAAIBJCN8AAACASQjfAAAAgEkI3wAAAIBJCN8AAACASQjfAAAAgEkI3wAAAIBJCN8AAACASQjfAAAAgEkI3wAAAIBJCN8AAACASQjfAAAAgEkI3wAAAIBJ6lT1AQAAgOsXXL+uAvy9/9u5xRLs1fkKCot14fxlr855I9BP3GiEbwAAarAA/zoa9IS1qg+jQh/OGaILVX0QlUA/caOx7AQAAAAwCeEbAAAAMAnLTgCgElgHCgDwhgp/J9m3b5+ys7O1fft2nTx5Ug0aNFCXLl00efJktWrVyqhLTk7Wjh073PYfMGCA5s2b5zLmcDi0YMECWa1WnT9/Xh06dNCUKVMUGxvrtv+uXbs0e/ZsHTx4UEFBQerfv7+eeOIJ1a1b93rOFwCuC+tAAQDeUGH4fvPNN7Vr1y7Fx8crLCxMdrtdy5YtU0JCgtasWaPbb7/dqL3ttts0efJkl/2bNWvmNmdqaqo2bNigMWPGqFWrVsrOztaECROUmZmpLl26GHU2m00pKSlq166dUlNTlZOTo7ffflsnTpzQokWLfsl5AwAAAKarMHynpKQoPT1dfn5+xtiAAQM0aNAgLVmyRLNmzTLG69evryFDhpQ73759+7R27VpNnz5dKSkpkqSEhAQNHDhQ6enpWrZsmVE7d+5cNWjQQJmZmQoMDJQkNW/eXM8884y2bt1a5pVyAAAAoLqq8IbL6Ohol+AtSa1bt1b79u119OhRt/ri4mJdvHjxmvOtX79evr6+SkpKMsb8/f2VmJionTt3Kjc3V5KUn5+vLVu2KCEhwQjekjRkyBDVq1dP69atq/jsAAAAgGrkup524nQ69eOPPyokJMRl/OjRo+rcubOio6PVq1cvLVq0SCUlJS41NptNbdq0cQnUkhQVFSWn0ymbzSZJOnTokIqLixUZGelS5+fnp/DwcKMOAAAAqCmu69b9Dz74QKdOndKUKVOMsRYtWujOO+9UWFiY8vPz9dFHH2nevHk6efKknnvuOaPObrerSZMmbnNaLBZJMq582+12l/Gf1+7Zs+d6Dl2NGgVd1341nbefqHCzu5n7eTOfe03Be+Q99NK76Kd30U/vMbOXHofvo0eP6rnnnlNMTIzL+u4XX3zRpW7o0KF6/PHHtWrVKqWkpKht27aSpIKCAvn6+rrN6+/vL0kqLCw06iS5LXkprS3d7qnTp/NVUuK8rn3LUlM++HZ79X/+QU3ppVQz+nkjWCzBN/W51xTV/T2il95FP72LfnpXTemnN3vp41Or3Iu9Hi07sdvtevjhh3XLLbdowYIF8vEpf/dx48bJ6XRq+/btxlhAQICKiorcaktDd2kIDwgIkHT1sYRl1ZZuBwAAAGqKSl/5vnDhgiZMmKALFy7o3XffLXM5yM81bdpUkpSXl2eMWSwWY2nJT5UuM2ncuLFR99Pxn9eW1gEAAAA1RaXCd2FhoSZOnKhvvvlGf/7zn40lJBU5fvy4JKlhw4bGWIcOHZSZmamLFy+63HS5d+9eY7skhYaGqk6dOtq/f7/uvfdeo87hcMhms2nQoEGVOgbgZsU3MgIAUP1U+DvzlStXNHnyZO3Zs0evvfaaOnfu7FaTn58vPz8/l/XZV65c0RtvvCEfHx+X53HHx8fr7bff1urVq43nfDscDmVlZSk6Otq4GTM4OFixsbGyWq16+OGHjaButVp16dIlxcfH/6ITB37t+EZGAACqnwrD96xZs/TZZ5+pT58+OnfunKzW//xmHhgYqLi4OB04cEBPPPGEBg4cqJYtW+rSpUtat26d9u/frwkTJqhFixbGPp06dVJ8fLzS09Nlt9vVsmVLZWdn6+TJk5o5c6bLa0+ZMkUjRoxQcnKykpKSlJOTo4yMDPXu3Vs9evTwYhsAAACAG6/C8P3VV19Jkj7//HN9/vnnLtuaNWumuLg43XbbbYqOjtaGDRv0448/ysfHR+3bt9esWbM0dOhQtznT0tI0f/58Wa1W5eXlKSwsTIsXL1ZMTIxLXUREhDIyMpSenq6ZM2cqKChIw4YN09SpU3/JOQMAAABVosLwnZmZWeEkLVq00Msvv1zpF/X399e0adM0bdq0Cmu7du2qFStWVHpuAAAAoLq6rm+4BAAAAOA5wjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgkgrD9759+/R///d/GjBggDp37qy7775bU6ZM0bfffutWu2vXLo0cOVKdOnVSz5499cILL+jy5ctudQ6HQ7Nnz1avXr0UFRWlYcOGaevWrWW+fmXnBAAAAKq7CsP3m2++qU8//VQ9evTQ008/rWHDhmnHjh1KSEjQ0aNHjTqbzaaUlBQVFhYqNTVViYmJWrlypaZMmeI2Z2pqqpYuXarBgwfr6aeflo+PjyZMmKDdu3e71HkyJwAAAFDd1amoICUlRenp6fLz8zPGBgwYoEGDBmnJkiWaNWuWJGnu3Llq0KCBMjMzFRgYKElq3ry5nnnmGW3dulWxsbGSrl5JX7t2raZPn66UlBRJUkJCggYOHKj09HQtW7bMeJ3KzgkAAADUBBVe+Y6OjnYJ3pLUunVrtW/f3rjynZ+fry1btighIcEIyZI0ZMgQ1atXT+vWrTPG1q9fL19fXyUlJRlj/v7+SkxM1M6dO5Wbm+vxnAAAAEBNcF03XDqdTv34448KCQmRJB06dEjFxcWKjIx0qfPz81N4eLhsNpsxZrPZ1KZNG5dALUlRUVFyOp1GrSdzAgAAADVBhctOyvLBBx/o1KlTxtpru90uSbJYLG61FotFe/bsMX622+1q0qRJmXWSjCvfnszpiUaNgq5rv5rOYgmu6kP4VaGf3kU/vYt+eg+99C766V3003vM7KXH4fvo0aN67rnnFBMToyFDhkiSCgoKJMlteYp0dUlJ6fbSWl9f3zLrJKmwsNDjOT1x+nS+Skqc17VvWWrKB99uv1DVh1ChmtJLiX56G/30rureT3rpXfTTu+ind9WUfnqzlz4+tcq92OvRshO73a6HH35Yt9xyixYsWCAfn6u7BwQESLr6CMGfKywsNLaX1hYVFZVZJ/0nhHsyJwAAAFATVPrK94ULFzRhwgRduHBB7777rstykNJ/L10q8lN2u12NGzd2qS1dWvLzOklGrSdzAgAAADVBpa58FxYWauLEifrmm2/0xhtvqG3bti7bQ0NDVadOHe3fv99l3OFwyGazKTw83Bjr0KGDjh07posXL7rU7t2719ju6ZwAAABATVBh+L5y5YomT56sPXv2aMGCBercubNbTXBwsGJjY2W1Wl1CtdVq1aVLlxQfH2+MxcfHq6ioSKtXrzbGHA6HsrKyFB0dbdyM6cmcAAAAQE1Q4bKTWbNm6bPPPlOfPn107tw5Wa1WY1tgYKDi4uIkSVOmTNGIESOUnJyspKQk5eTkKCMjQ71791aPHj2MfTp16qT4+Hilp6fLbrerZcuWys7O1smTJzVz5kyX167snAAAAEBNUGH4/uqrryRJn3/+uT7//HOXbc2aNTPCd0REhDIyMpSenq6ZM2cqKChIw4YN09SpU93mTEtL0/z582W1WpWXl6ewsDAtXrxYMTExLnWezAkAAABUdxWG78zMzEpP1rVrV61YsaLCOn9/f02bNk3Tpk3z2pwAAABAdXdd33AJAAAAwHOEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAklQrfubm5Sk9PV3Jysrp06aKwsDBt377dre6ee+5RWFiY26/09HS32vPnz+vZZ59V9+7d1blzZ40ZM0Y2m63M19+0aZOGDh2qjh076u6779bChQtVXFzs4akCAAAAVatOZYqOHTumJUuWqFWrVgoLC9Pu3buvWRsREaGxY8e6jIWGhrr8XFJSooceekiHDx/WuHHjFBISouXLlys5OVlZWVlq2bKlUfvFF19o0qRJ6t69u5599lkdPnxYr776qs6ePatnn33Wk3MFAAAAqlSlwndERIS2bdumkJAQbdy4UZMmTbpmbdOmTTVkyJBy51u/fr12796tV199VXFxcZKk/v37q1+/flq4cKHS0tKM2rS0NN1xxx166623VLt2bUlSYGCgFi9erOTkZLVu3boypwAAAABUuUotOwkKClJISEilJ3U4HLp8+fI1t3/yySdq3Lix+vbta4w1bNhQ/fv318aNG1VUVCRJOnLkiI4cOaLhw4cbwVuSHnjgAZWUlGjDhg2VPiYAAACgqnn9hsu///3v6ty5szp37qy4uDitXLnSrcZmsykiIkK1atVyGe/YsaMuXryo7777TpJ08OBBSVJkZKRLXZMmTdS0aVNjOwAAAFATVGrZSWWFhoaqa9euat26tc6ePatVq1bp//2//6e8vDw99NBDRp3dblf37t3d9m/cuLGkqzd43n777bLb7ZIki8XiVmuxWJSbm+vxMTZqFOTxPr8GFktwVR/Crwr99C766V3003vopXfRT++in95jZi+9Gr4XLVrk8vN9992nBx54QK+99ppGjhyp4OCrJ1ZQUCA/Pz+3/UvHCgoKXP5ZVq2/v3+5S1uu5fTpfJWUOD3e71pqygffbr9Q1YdQoZrSS4l+ehv99K7q3k966V3007vop3fVlH56s5c+PrXKvdh7Q5/zXbt2bY0dO1aXL192eUJKQECAHA6HW33pWEBAgMs/y6otLCw0tgMAAAA1wQ3/kp2mTZtKkvLy8oyxay0ZKR0rXX5SutykdPnJT9ntdqMOAAAAqAluePg+fvy4pKtPMynVoUMHHThwQE6n6/KPffv2qV69esZzvsPDwyVJ+/fvd6k7deqUcnJyjO0AAABATeC18H3u3DmVlJS4jBUWFuqtt95SYGCgOnfubIzHx8crNzdXmzZtMsbOnDmj9evXq2/fvvL19ZUktW/fXm3bttXKlSt15coVo/bdd9+Vj4+P7r33Xm8dPgAAAHDDVfqGy9dee02SdPToUUmS1WrVzp07Vb9+fY0ePVqfffaZFi1apH79+qlZs2Y6d+6csrOz9c033+h///d/FRgYaMzVr18/de7cWU8++aTxDZfvvvuuSkpK9Oijj7q87pNPPqlHHnlE48eP14ABA3T48GEtW7ZMw4cPV5s2bbzRAwAAAMAUlQ7fCxYscPn5vffekyQ1a9ZMo0ePVmhoqNq2bSur1aozZ87Iz89PERERSk1NVZ8+fVz2rV27thYvXqy0tDRlZmaqsLBQHTt21EsvvaRWrVq51Pbp00cLFy7UwoUL9fzzz6thw4Z65JFH9Ic//OF6zxkAAACoEpUO34cOHSp3e2RkpNujBstzyy23aMaMGZoxY0aFtXFxccbX0AMAAAA11Q2/4RIAAADAVYRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSEbwAAAMAkhG8AAADAJIRvAAAAwCSVCt+5ublKT09XcnKyunTporCwMG3fvr3M2k2bNmno0KHq2LGj7r77bi1cuFDFxcVudefPn9ezzz6r7t27q3PnzhozZoxsNtsvmhMAAACozioVvo8dO6YlS5bo1KlTCgsLu2bdF198oUmTJumWW27Rs88+q7i4OL366quaOXOmS11JSYkeeughrV27VqNHj9Yf//hHnT59WsnJyfruu++ua04AAACguqtTmaKIiAht27ZNISEh2rhxoyZNmlRmXVpamu644w699dZbql27tiQpMDBQixcvVnJyslq3bi1JWr9+vXbv3q1XX31VcXFxkqT+/furX79+WrhwodLS0jyeEwAAAKjuKnXlOygoSCEhIeXWHDlyREeOHNHw4cONkCxJDzzwgEpKSrRhwwZj7JNPPlHjxo3Vt29fY6xhw4bq37+/Nm7cqKKiIo/nBAAAAKo7r91wefDgQUlSZGSky3iTJk3UtGlTY7sk2Ww2RUREqFatWi61HTt21MWLF42lJ57MCQAAAFR3lVp2Uhl2u12SZLFY3LZZLBbl5ua61Hbv3t2trnHjxpKu3uB5++23ezRnZTVqFOTxPr8GFktwVR/Crwr99C766V3003vopXfRT++in95jZi+9Fr4LCgokSX5+fm7b/P39dfnyZZfasupKx0rn8mTOyjp9Ol8lJU6P97uWmvLBt9svVPUhVKim9FKin95GP72ruveTXnoX/fQu+uldNaWf3uylj0+tci/2em3ZSUBAgCTJ4XC4bSssLDS2l9aWVVc6VlrryZwAAABAdee18F26NKR0qchP2e12Y0lJaW1ZS0ZKx0prPZkTAAAAqO68Fr7Dw8MlSfv373cZP3XqlHJycoztktShQwcdOHBATqfr8o99+/apXr16atmypcdzAgAAANWd18J3+/bt1bZtW61cuVJXrlwxxt999135+Pjo3nvvNcbi4+OVm5urTZs2GWNnzpzR+vXr1bdvX/n6+no8JwAAAFDdVfqGy9dee02SdPToUUmS1WrVzp07Vb9+fY0ePVqS9OSTT+qRRx7R+PHjNWDAAB0+fFjLli3T8OHD1aZNG2Oufv36qXPnznryySc1btw4hYSE6N1331VJSYkeffRRl9et7JwAAABAdVfp8L1gwQKXn9977z1JUrNmzYzw3adPHy1cuFALFy7U888/r4YNG+qRRx7RH/7wB5d9a9eurcWLFystLU2ZmZkqLCxUx44d9dJLL6lVq1YutZWdEwAAAKjuKh2+Dx06VKm6uLg44yvjy3PLLbdoxowZmjFjhtfmBAAAAKozr635BgAAAFA+wjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEsI3AAAAYBLCNwAAAGASwjcAAABgEq+G7+3btyssLKzMX0ePHnWp3bVrl0aOHKlOnTqpZ8+eeuGFF3T58mW3OR0Oh2bPnq1evXopKipKw4YN09atW7152AAAAIAp6tyISceOHauIiAiXsSZNmhj/brPZlJKSonbt2ik1NVU5OTl6++23deLECS1atMhlv9TUVG3YsEFjxoxRq1atlJ2drQkTJigzM1NdunS5EYcPAAAA3BA3JHx369ZNcXFx19w+d+5cNWjQQJmZmQoMDJQkNW/eXM8884y2bt2q2NhYSdK+ffu0du1aTZ8+XSkpKZKkhIQEDRw4UOnp6Vq2bNmNOHwAAADghrhha77z8/NVXFxc5viWLVuUkJBgBG9JGjJkiOrVq6d169YZY+vXr5evr6+SkpKMMX9/fyUmJmrnzp3Kzc29UYcPAAAAeN0NCd9//OMfFRMTo06dOmncuHE6dOiQse3QoUMqLi5WZGSkyz5+fn4KDw+XzWYzxmw2m9q0aeMS0iUpKipKTqfTpRYAAACo7ry67MTX11f9+vVT7969FRISokOHDuntt9/WAw88oDVr1qhNmzay2+2SJIvF4ra/xWLRnj17jJ/tdrvLWvGf1kniyjcAAABqFK+G7+joaEVHRxs/9+3bV/fcc4/uv/9+LVy4UHPmzFFBQYGkq1e6f87f39/YLkkFBQXy9fUts06SCgsLPT7GRo2CPN7n18BiCa7qQ/hVoZ/eRT+9i356D730LvrpXfTTe8zs5Q254fKnOnTooNjYWG3btk2SFBAQIOnqIwR/rrCw0NheWltUVFRmnfSfEO6J06fzVVLi9Hi/a6kpH3y7/UJVH0KFakovJfrpbfTTu6p7P+mld9FP76Kf3lVT+unNXvr41Cr3Yq8pX7Jz6623Ki8vT9J/loyULj/5KbvdrsaNGxs/WyyWMpeWlO7701oAAACgujMlfB8/flwhISGSpNDQUNWpU0f79+93qXE4HLLZbAoPDzfGOnTooGPHjunixYsutXv37jW2AwAAADWFV8P3mTNn3Mb++c9/avv27erVq5ckKTg4WLGxsbJarS6h2mq16tKlS4qPjzfG4uPjVVRUpNWrVxtjDodDWVlZio6OLvNmTAAAAKC68uqa78mTJ6tu3brq0qWLQkJC9PXXX2vlypUKCQnRo48+atRNmTJFI0aMUHJyspKSkpSTk6OMjAz17t1bPXr0MOo6deqk+Ph4paeny263q2XLlsrOztbJkyc1c+ZMbx46AAAAcMN5NXzHxcXpww8/VEZGhvLz89WwYUMNHDhQjz76qG677TajLiIiQhkZGUpPT9fMmTMVFBSkYcOGaerUqW5zpqWlaf78+bJarcrLy1NYWJgWL16smJgYbx46AAAAcMN5NXyPGTNGY8aMqVRt165dtWLFigrr/P39NW3aNE2bNu2XHh4AAABQpUy54RIAAAAA4RsAAAAwDeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwCeEbAAAAMAnhGwAAADAJ4RsAAAAwSY0I3w6HQ7Nnz1avXr0UFRWlYcOGaevWrVV9WAAAAIBHakT4Tk1N1dKlSzV48GA9/fTT8vHx0YQJE7R79+6qPjQAAACg0qp9+N63b5/Wrl2r//mf/9GTTz6p4cOHa+nSpbr11luVnp5e1YcHAAAAVFq1D9/r16+Xr6/v/8femcfVlP9//NVtvy1SWmgj5bTvpUWlRKEYhmlmLINBY8kYZsEMhvmOMRjEYJgYkZGlbKVUlEokS1mqUYmQlDbtde99//7od8+4k+98Z6GS83w85jE559xzP5/P/Szvz/vzXjBx4kT2mry8PCZMmICrV6+ivLy8C0vHwcHBwcHBwcHB8dfp9sJ3Xl4eBgwYACUlJYnr1tbWICLk5eV1Uck4ODg4ODg4ODg4/h4yXV2A/0VFRQW0tbU7XNfU1ASAv6355vGkXkq5nkert+JLf+fL5lXU+1XwOrQlwLXny4Zrz5fL69CeXFu+XLj2fLlw7flyeR3a82W25f96lxQR0Uv7tleAr68vjI2N8dNPP0lcf/DgAXx9fbF8+XJMnjy5i0rHwcHBwcHBwcHB8dfp9mYnCgoKaGtr63C9paUFQLv9NwcHBwcHBwcHB8frQLcXvjU1NV9oWlJRUQEA0NLS6uwicXBwcHBwcHBwcPwjur3wbWpqiuLiYjQ0NEhcz8nJYe9z9JGiJwAAIABJREFUcHBwcHBwcHBwvA50e+Hb398fbW1tOHLkCHuttbUV0dHRsLe3f6EzJgcHBwcHBwcHB0d3pNtHO7GxsYG/vz82bNiAiooKGBgY4NixYygtLcV3333X1cXj4ODg4ODg4ODg+Mt0+2gnQLtz5ebNm3Hq1CnU1taCYRgsWrQIbm5uXV00Dg4ODg4ODg4Ojr/MayF8c3BwcHBwcHBwcPQEur3NNwcHBwcHBwcHB0dPgRO+OTg4ODg4ODg4ODoJTvjm4ODg4OixlJeXIyUlpauLwcHB0U05f/48rly5gs60wuaE7zeAJ0+edHURODg4ODqdpqYmjB8/HjExMQAAoVDYxSXi4OgaOPe+jhw/fhyjRo1CcHAwbty4AYFA0GnfzQnfPRgiwrx58/DFF1+gubm5q4vDwfG3SE5OxurVq1FYWNjVReF4TXny5Anq6upgaWkJAJCWlu7iEnH8Ga2trV1dhB5FVVUV9uzZAwCQkpLq4tJ0Hw4fPgxvb2+sWrUKZmZm2LlzJ9566y3Iysp2Whm6fZxvjn+GSCRCS0sLcnJy4O/vDwUFBRARNwD/Jc+ePUN+fj50dXXRu3dv8Pl8iEQi8HjcPvZlceLECfz0008oLi6Gt7c3FBQUurpI3YLm5ma2Lbix3JGqqiqoq6tDJBJBSkoKUlJSEAqFaGlpgYmJCQCu3boreXl5OH36NKqqqjBkyBCMHDmSm1f/Bffv30doaCji4+OhpaWFESNGQE9Pr6uL1aU0Nzdjx44diIiIYDOm//zzz3BycoK8vHynzwuc8N3DEAqF4PF44PF4qKmpQV1dHYyNjQFwO99/Q3FxMX788UdkZGRAJBJBRkYGDg4OWLduHSccviR+/fVX7N27Fw8ePMDo0aOxZMkSmJubQ1NTs6uL1qXExMQgNjYWPB4PAwcOxLRp06Curt7VxepyxIJ0ZWUlPvjgAzQ2NuLbb7+Fg4MD5OTkAAD5+fmQl5dnNVrcHNi9uHHjBjZu3Ihr167B0NAQMjIy4PP5GDlyJCd4/wNu3LiB0NBQXLhwAQzDYNmyZXB1dYWOjk5XF63LICKsW7cO+/fvh5qaGqZOnQo5OTls374dOTk58PDwgFAo7PRTMU74fo0hIhARqyVcunQpuygTEaqqqgAAqqqq7DVu8fl73Lt3D99//z2Sk5NhamqK6dOnQ05ODikpKUhISICuri6++OILrm3/IS0tLaw2orm5GQKBAF5eXli9ejX4fH5XF69L2bt3L8LCwlBfX49BgwahuroaZ8+eRWFhIVauXAltbe2uLmKXIh5vGhoamDlzJiIjIzF37lyMHTsWc+bMgba2NkQiEVpbW8EwTBeXluOP3L17F19++SXU1dXxzTffwNjYGEZGRlBUVOzqor12VFVV4fvvv8eJEyfg7u6Ob775Bi4uLujbty9kZN5MMa+6uhr19fXQ19dHr1698NFHH8HX1xempqaorKxEdnY2wsPDMX/+/C4xR3szf5UegnjxKS0tRVxcHG7cuIF58+Zh1KhRkJGRQUVFBVpaWmBsbMwJh/+QQ4cOITk5Ge+//z4WL14MAFBSUoKfnx9CQkIQGRmJkJCQN15Q/CdcuHABH3/8MeTk5DBp0iS4uLggISEBUVFRqK2tBZ/PfyP7bVlZGT755BNcv34dkyZNgr+/P4yMjCAlJYUtW7bg0KFD8PDwwDvvvPPG2TDX19fj0KFDePjwITQ0NGBiYgJvb2+89dZbcHd3x+HDh7Ft2zZcvHgR3377LbKysmBsbIympib06tWrq4vP8f+IRCJs374d5eXl+Oabb2BlZcX2ZfHp7Zs27v8N0tLSqKyshKKiIj799FOYmZl1eKa1tZU9EerJ3LhxA+Hh4SguLoaysjJmz56Njz76SKL+GhoaGDZsGM6fP4/9+/djypQpEAgEnbpRkf7666+/7rRv4/jX3L9/H9HR0SgpKYFIJIKWlhZ8fHzg7OyMu3fvIiwsDPn5+XB0dEReXh6uXLmCd955hzum/oskJSUhJyeHnbxUVVWRk5ODmpoavP/++5CTk4NIJIKqqipu376N7Oxs2NnZoX///l1b8NeE8vJyZGdnQ1paGvr6+tDS0kJwcDB8fX0xYMAAKCoqIjExES0tLfDw8HgjF+C2tjYUFRXh3r17WL58OWxsbMDn86GoqAh1dXWkpaWhra0No0ePZu2bezqNjY3YsWMHFixYgDt37qCmpgapqak4efIkUlNTYWtrCwMDAzg7O8PS0hJ5eXk4ePAgrly5AgMDA0ybNq2rq/DG0traiqysLNTU1KBXr16QkZFBbW0t9uzZAwsLC0yfPp210QfACd5/gZSUFFy8eBFaWlpQVlaGgoICeDweUlJSoKCgADc3N/bZR48eYePGjYiMjIS7u3uPVRQ9evQIX331FUJDQyEnJweGYTBgwAA4Ojqid+/erGAt3typqqoiNzcXKSkpmD59eqebOXGa79eEnJwcbNy4EVlZWVBRUUFtbS1UVVWxePFiBAUFwdHREY6Ojti1axf27t2LkJAQVFVVQV9fnz1+5Wzo/pyqqiqEh4ejoKAAXl5eUFdXh7W1Ndzd3REeHo5Lly7BxcWFDVdWWVkJdXV1GBkZdXHJuz+3bt3C9u3bkZeXBwAwMzPD2rVrERQUJPGciYkJhg0bhujoaMyaNQt9+vTpiuJ2GmJtzPMa/t69e8PPzw8nTpxAQkICTE1NWa2MkpISamtr8eDBAwB4Y8Z0dHQ0IiMjMXnyZPj7+0NHRwfy8vIICwtDeHg4li5dii+++AKDBw+Gl5cXXF1dsW/fPvz0009obm7GkydPoK2t/UaepHQV1dXV2LlzJ5KSktDc3AxtbW2sWLGC3UzyeDw8evQIpaWl6NevH549e4YnT56Ax+Ph8ePHGDBgAHR1dbu6Gt2G1tZWhIWF4ejRowCAx48f4/Hjx1i4cCF4PB5cXV3h6OiIqKgofPrppyguLsbWrVuRkJCAfv36YdasWT3apOfXX39FXl4ePvvsMwwZMgQ6Ojov1PSLT1j09fUxYsQIfPfdd4iJiUFAQECnar85zXc3p7S0FO+99x527doFPT09zJkzB++//z68vLxw+fJlXLp0CY6Ojqz9p4ODA1xcXFBYWIhr165BXl4eU6ZMgaysLLfw/IHW1laUlJRARkYG8vLyUFRUBJ/Px+nTp6GnpwcLCwsAgJycHC5duoSSkhIEBgaisbERW7duRVRUFPz8/DBmzJg37vj/r9La2ootW7ZgzZo1UFZWxqhRozBkyBBMmDABWlparPAo7puKiorg8XiIi4uDgoICnJycurgGr4Z79+5hxYoVuH37NhwcHNjxCbSbkykpKaG0tBTx8fEYOXIkazKRkJCA7OxsTJ8+HVZWVhKf6UkIBAK2b+Tm5mLZsmVwdHTE8uXL0bdvX/D5fMjJycHBwQF6enr49ddfUVNTAw8PD1YL6ODgAJFIhNOnT8Pc3BwMw/S4dupuiMdxQkICFixYgJKSEvj6+mL48OEIDAxkQz7Kysri2bNniI2NxYULF5CVlYWIiAgcPnwYu3fvxvHjx3HkyBEoKChgwIABb3y0rocPH+Kzzz7DhQsXMGzYMIwbNw7Tpk3D2LFj2bWHz+ejsbERKSkpOHr0KHbu3AlZWVksXLgQCxcuhKOjY48yO6mpqWGDHVy/fh3/+c9/EBAQgLlz56JXr16QlpaGSCSCQCDosD6LTwxVVFRw/fp1XLp0CZMmTepUZQYnfHdTxBONnJwc1qxZAxsbG6xcuRIeHh7o168fjI2NoaioiNjYWGhra8PZ2Zn9rJaWFvz9/fHkyRPk5+fD0NAQJiYmb/Tk9TxVVVXYsGED1q1bh7i4OGzatAkmJiYYOHAg+Hw+bt++jaysLIwePRry8vLQ1tbGvXv3cObMGRQXF+Prr79GUVERFi5ciMWLF3OC95+QlZWF0NBQBAYGYuHChRg6dCisra2hrq4uMdFJSUmx/VNZWRn3799HYmIiJk6cCHl5+S6swathx44diI6OxoMHD1BWVoahQ4eyY1O8CZGVlcWxY8egpaWF+vp6LF26FAcPHkRjYyMaGxuho6MDAwODHjWmKysrMXv2bNTU1MDe3h4AcO7cOcTHx2P58uUdzLukpaXBMAyKi4uRmJiI/v37w9zcnD1atrGxwe7du9HW1gYXFxcoKipy8+ArREpKCjU1Nfj666+ho6ODr776Cv7+/rC3t4euri5kZGRYExMzMzNoaGggMzMT9+/fR3V1NaytrTF69GgMGjQIz549Q2JiItTU1GBra/tG/mbifhwWFoZz587hiy++wLvvvgtLS0v07du3g/JCTU0N+fn5uHPnDjZt2oSQkBBYWVlBWVm5x7TfkydPMGHCBOTn58PX1xdSUlK4d+8ejh49ihkzZrDR3YD2/vj8+ixuJ3FbqKuro6amBnFxcbC2toahoSHb5q+aN+PM8jVCnI1SrNGSl5fH9OnT8dtvv+HRo0cA2m1CAcDOzo79t0gkYt8hNosQ2znGx8ejra3tjTmifhEikQhtbW04ePAgAgMDkZ6eDl9fX0ycOBHz589HY2MjAEBHRwdvvfUWfvvtN6SnpwMAZGRk4O3tDT09PcTHx2Px4sXYv38/pk6dyr5bTEtLC549ewbgzcwolpaWhtraWgDtcVWPHTsGAPj000/ZUGLA7334+TYST4iamprw9/dHVVUVTp482ZnFf6UQETs2LS0tIScnh88//xyRkZH48ccfUV9fDykpKTbLmoWFBXx8fLB27Vp89NFHUFFRwbp16/Cf//wHbW1tmDFjBjZt2sTOGT0he2NraytEIhGOHz+O2tpaEBHu3bsHAKx51/N9Rvz33LlzAbTbwra0tEBGRgZtbW2QlZXFe++9h9TUVFy9ehVAzzsl6GrOnz+PCxcusP0vNjYWt27dwpQpU2BjYwMlJSUAYPu1eKPN5/MxefJkxMTEIDIyEufOncP27dsxd+5cfPnll9iyZQtkZWVx/vx5VFRUdFn9Opv8/HxERUUB+N1EIiYmBkOHDkVgYCCUlZXZZ59fewBAT08PQ4YMARGhuroaKioq//XZ1xVFRUU4OTkhPT2dNb8zMTGBuro6YmNjUVJSgpqaGpw7dw7Hjh3D1q1b8e233+Ly5csS7xG3h4eHBwwNDbF9+3YAnWfK9+ZKY92I5uZmbNu2DZMmTcL8+fMRGBiIyMhI9v6sWbPQ3NyMM2fO4NmzZ2zM2sTERKioqMDc3Bw8Ho9diKSlpUFEGDhwIIYMGYJr164hIyMDQM8ZgH8XHo+HW7duISwsDA4ODvjhhx8wf/58TJ8+HcHBwRg7diz7nIODA6ysrHDgwAFWKB88eDCsra0hJSUFNzc3DBgwQOLdQHuUijlz5mDTpk0A3rxF/uDBg5g1axauXbsGAFBQUMDDhw+hrq4usXgKBIL/GndZ3IdtbW3h4uKCffv2vdZZ744ePYrz588DkNTCKCgoQElJCS4uLli8eDGOHj3K9hvxBkVTUxN+fn5QVFTElClTsGnTJgQEBGDChAkIDQ1FcHAwdu7ciWXLlqGhoaFHnMDo6Ojg/fffR1FREc6ePSuxGRH7CzyPWJAbOHAgHB0dUVBQgNLSUgC/j8vg4GCIRCKcO3cO9fX1AN7MjfHL5vjx4xg5ciSCg4Nx+/ZtdpyKhW0tLS2J58X9uqmpSWLc8/l86OvrQ0lJCUKhkF2j9PT0wDAMKioqoKam9sb8ZkuWLMGGDRtw+/ZtAO2mp2pqaqygKaa2thY8Hg+VlZVoaWlhr3t5ecHU1BTh4eEAfh8H4v83Nzfj6dOnALr/OBBn5n6+nKqqqhg9ejTa2toQHR0NAOjTpw9mzJiBhIQEjBs3Dl5eXpg7dy6WLl2Kbdu2scqybdu2se8R90EzMzP4+/sjOzsbN27cYJNzvWo4s5MupKWlBb/88gvmzZuHgoICWFpaQldXF05OTlBQUICpqSmICEpKSrh//z7Onz8PNzc3qKio4IcffsD+/fvR0tICR0dHDBgwgD2eJyLWwVJbWxvR0dGQkpKCj4/PGycQim27hEIhFi1ahJaWFmzcuBFGRkYdvJ/FzyopKUEgEODQoUNwcHCAoaEhZGVlIRQKkZaWhqamJnh5eUnYpQLtm54VK1bg6dOn8PX1ZeOrvyn06tULKSkpqK+vh4ODAxQVFSW031paWkhOTkZSUhKOHTuGLVu24Pbt2+DxeBKbGbEtXltbG44dOwYjI6PXKk5zU1MTtmzZgrlz5yIzMxNubm4YOHCghGlNcXEx0tLS4OXlhaFDh4LP5+PHH3+EvLw8Bg0axI5lJSUl3Lp1C4WFhXj77bdZm00lJSW4urpCIBAgISEBZ86cgZaW1mvv/CslJQU+n49bt27h2rVrmDBhAtra2nD69GlYWlrCysqqg2ZKSkoKIpEIBQUFuHz5MqZOnQoVFRXweDwIhULw+Xzk5+cjNjYWnp6e6Nev3xs3D75M9u3bh3nz5iE+Ph6urq5YvHgxvL292fmutbUVZ8+eRUFBAfr06YP8/HzExcXh1KlT2LlzJ86ePYvm5mYYGhpCTk6OnX/Fzm7i3+bKlSvYs2cPBgwYIGHb3FNpa2uDtLQ0lJWVkZycDD6fD1dXVygpKeHevXuIi4tDY2MjHjx4gOPHj+PUqVP48ccfsXPnTjaZ1IABA6Curo7y8nIkJiZi4MCBbKjhxsZG3LlzB7/++is2bNiAgICAbuuA2dbWhvnz5yMrKwtubm4d0r4rKSnh0aNHSEhIwKhRo6CsrAx7e3uoqamhT58+0NTUhLu7Oz7++GO8/fbb8Pb2xm+//YbU1FQ26ZB43uDxeJCVlcXJkydRXFyMcePGdY72mzi6jF9//ZXc3d1p1apVdOvWLXr27FmHZwQCARERFRcXE8MwNHLkSLKwsKDhw4fTypUradKkSeTl5UWTJ0+m69evExGRSCSSeEdwcDDZ2tpSRkbGq69UNyAjI4PmzZvH/lskEtH9+/fJ0dGR1qxZQ0REra2tf/qOgoICGjVqFAUHB1NLSwsREdXU1NC8efPIwcGBqqur2Xc/z/Xr16mysvJlVqfb8eTJE2psbCQiyfoLBALaunUr2dra0uXLl9nrU6dOJYZhJP7z9vYmLy8vYhiGbGxs6MKFC+y7hEIhERHdu3ePJk2aROPGjWOvdWcqKiroyy+/JHNzc/Lw8KCNGzdSdnY221bP8+TJE2IYhi5evMheW7t2LXl7e9O3337LXhMIBBQVFUUMw1BsbKzEdSKitrY2OnPmDI0dO7bHjG+BQECRkZHEMAydPXuWCgsLadiwYTRt2jQqKSkhIsl+J/57xYoVZGpqSvn5+RLvIiIqKyujxMTETqxFz+PatWtkYWFB1tbWtHLlSrp8+TI7Dz5Pc3Mz7du3r8OYd3d3J3d3d7KzsyOGYej7779nPyP+DUUiET1+/JgSExPpvffeoxEjRlBOTk6n1bEzSU1NpatXr7LrhbivEhG99dZb9NZbb1FhYSEREeXl5dHixYs7tOeMGTMoMDCQGIYhV1dXOnfuHBER5eTk0KhRo2jixIkkFAopKyuLVq9eTc7OzuTo6EgrV678n2tgVyGe6ydPnkzu7u6UlZVFRB3X2uTkZLKysqKwsLAO73hR3cTz6I8//ijxPURELS0tdOrUKSouLn5Z1fifcMJ3J/HHjpOTk0OOjo40b968DkL3HzuO+LMhISHEMAytX7+eamtrqa2tjYRCIaWmppKzszO5urpSeno6+7xYaExJSaFFixZRaWnpq6pet+D06dMkEAgoNDS0g7CSlZVFDMNQaGjoX3pXS0sL7dy5k8zMzOjKlSvs9ejoaHJwcKDNmzcTUbvw8yYRFRVFVlZWdOjQISKiDkJxYWEhubi40IoVK6impoaIiEpLS+ncuXP0yy+/0C+//EJ5eXlUVlZGT58+pSNHjpCbmxvNmjWrw6ZFKBTSzp07iWEYunHjRudU8B+ya9cudlMhrmNTU9N/fb6kpIR8fHzop59+Yq9VV1fT7NmziWEYioiIYAWboqIimjBhAk2ePPlP39mTuHv3Lo0ZM4ZmzpxJlZWVtG7dOjI3N6f9+/dLbDyeJzg4mPz8/EgkEnWYbzn+GeI1hKh9zbK3t6dFixZReXn5//xsSkoKRUZG0vbt2ykjI4OePn1KZWVlVFhYSKNHjyZzc3N2M5WWlkYrV66kTz75hD788EOytramwMBASk1NfWV16wqamppo06ZN5OrqSvb29mRra0teXl6UnZ0t8VxERATZ2tpKzA9tbW2Um5tLsbGxlJmZSS0tLVRfX09ERImJicQwDM2aNYt9fsOGDWRlZUXvv/8+2dvbk6ur6wsF1e7A8/1M/HdeXh4xDEPr1q2T2JiIx3Z5eTnNmTOHhg8fLrF2iJ8VCAQkEAjY52/cuEEMw9Dq1atfeX3+Cpzw3Qmkp6fT48ePiej3jhUWFkbW1taUl5fHPvfHBSMtLY3i4uLYf9+8eZMYhqENGzaw18QLUHp6Or3zzjtkb29PmzZtemV16W40NjbSDz/8QDY2NuTl5UVFRUVUUFBAY8aMobfffpt9rqSkhGxtbWn9+vV/KjA//xvcvn2bvL29adGiReyG6MmTJxQUFEQMw0hMGD0dcbsUFxeTi4sLLViwgJqbmyXuEbX372+//ZacnZ3/VGMlniBrampowYIFZGdnR2VlZR2+r6ysjB49evTS6/MyyMzMpOXLl1NJSQmlpKSwY/P5heJ5nu8vT58+JR8fH9q9ezc9e/aMDh8+TAEBAWRjY0PDhg0jLy8vWrduHfu5iIgIYhiGkpKSOqVuXU1LSwvt3r2bTE1N6fLly1RUVER+fn7k5ubWoQ0aGxvpwIEDZGZmxgornPD971mzZg3Nnj2bHX+NjY20atUqcnd3l1i3iIiSkpLI19eXjh49+pfevWfPHnaTSdSuHJk4cSJNnTqVFi5cSCkpKS+3Ml1MU1MTfffdd2RhYUFDhgyhtWvX0tGjR2nNmjVka2tLQUFBVFRUxD7f0tJCI0aMoKCgIFZ2eNG69Xw/9/f3p9GjR7PzaEZGBnl4eNCIESPoyJEjr7iG/4x9+/bRtGnTaO7cubRhw4YOdZw0aRINGzaMXUv+qOw5ffo0mZmZ0eHDhyWu/3H8P3z4kBYuXEgODg7sqWxXzxGcw+UrJjs7Gx9++CFWrVoFoN3pRCAQ4ObNm5CVlYWenh77rNjWLTo6GhMnTsTMmTOxcOFC9r6lpSVsbW2RkJCAW7duSXyPu7s7duzYARMTE+zcufOFzkk9icrKSnz//fdwcnLCyZMnMWnSJGzZsgV6enowNjaGp6cnbt++jbS0NADt9vX9+/fHhQsXWAeuF/G8LWj//v0REBCA5ORkFBUVAWi3W546dSqbRYu6ucPKv+HSpUuIiYlBcnIyGhoaALS3ibe3N65cucK27fNtICcnh7Fjx6KtrQ0JCQmsw4zYiUooFErEXe3VqxdqampARKwzHPD776CtrY1+/fq9+sr+Dc6ePYvx48dj6tSpePDgAZqbm+Hl5QVDQ0NkZmZKOEY1NjZi586dGD16NLKzswG0t5eGhgZUVVURGRmJMWPGYNWqVdDX18fWrVvx888/w9PTE7t378bRo0dBRPDw8ICBgQEeP37cVdXuVOTk5ODu7g49PT3s378fRkZGWLZsGerr6/HJJ59gw4YNyMzMxNmzZ7F7926EhYXB29ubTdrE2XT/c8TjWVZWFmlpabh58yaA9igTY8eORV1dHc6fP4+mpibW6XLRokUwNDTEwIEDX/gucXQjMWKnTDU1NQCAo6MjfvrpJ2zevBmbNm2Cl5fXK61jZ/D8vHjr1i1ER0djwIAB+PnnnxESEoK3334bS5cuxcyZM3Hz5k3k5OQAaHdIl5OTw/jx41FQUICkpCQAkLB5FwgEEiEzy8rKUFNTg969e7PZrF1cXPDLL7/gzJkzmDBhQmdV+3/S1NSEzZs3w87ODlu2bIFAIEBeXh5+/vlnrFy5UmKOmzdvHh4+fMiuNX+0xba1tYWDgwMOHTrEXhO3e3NzM/Lz85GUlIT169cjNTUV77//PqysrAB0gzmiq6T+no54h1ZXV0fLli0jc3Nzevr0KXt/0aJF5OzsTDdv3pT4nPho79NPP6UffviBTE1NKTw8nL2flpZGDMPQjh07Xvh9ZWVlPVojW1hYyJrfMAxDU6dOpdzc3A6mOlevXqVhw4bR1KlTiahdk7Bq1aq/pD1MTExkj7GuXLlCDMPQkiVLXgu745fBvn37yNvbmzw9Pcnb25sYhqFdu3ax93Nzc8nJyYkWL178wjZpbm6mpUuXkqenJ925c+e/fs/Tp0/pl19+IXt7e1q7di0Rdb024s84efIkDR8+nMzNzWnOnDmUmJgoMd6OHDlCDMPQ0aNHqba2ltavX08ODg7k7u5Oq1evpidPnrDvamxspM8//5xsbGxo+fLllJGRQVVVVWx7lpeXs/18z549RERvjMmJmKamJtqyZQtZWFiwZkcXL16k4OBgsrS0JBsbG3JxcSFHR0dat24dewTP8fcRr03Pn9o8e/aMNTMRz4f19fW0aNEisrGxoSFDhpCtrS0tW7aMrl+//kLfhj/S0NBAqampNGzYMHrvvfd6rH9MWVkZ1dbWsuO5qqqKvvvuO3JwcGBNbcSI1/RvvvmGiH7XcFdXV9PQoUNp2rRpVFVV9cLvaWpqory8PFqyZAk5Ojp26xMDkUhEDx48IFdXV9Z89ubNm/Ts2TN6+PAhLVu2jCwtLTto6ceMGUMBAQH022+/EZGk9lsgEFBYWBgxDMPaxz979oy2bdtGNjY2NG7cOHJ2dqaRI0fS6dOnO6+yfwFO+H5JNDY2UmRkJN26dYu9Ju4kiYmJZGNjww4ugUBAv/zyC1lYWHRwAmppaaH79+8TEdGDBw9o0qRJ5OHhIfFMYGAgjRvHCuTUAAAgAElEQVQ3ju2M3VlgeVk8fvyYJk+eTAzD0Lhx4+jw4cM0YcIECggIoIKCAiJqb29xm4tEIvrmm2/I3Nycrl27RkTtDhoeHh4UFBTEToB/POaqqKggZ2dn2rZtGxER1dbW0qFDhyg3N7ezqtplFBYW0vTp08nV1ZWWL19O8fHxdOPGDSorK5Po10Tt/gdeXl504cIFIup4HJiZmUkWFhbssTJR+8Kdn59PGRkZFBUVRYsWLSJbW1uaPXs2O3F2V3Jzc8nOzo48PDwoPT2dKisrXzju3N3dydnZmWxsbMjHx4e2b99OxcXFL3QAmjt3roRT1R+pq6ujiIiIHr2Z/l/k5OTQkCFDaOnSpey1xsZGysvLo9TUVEpKSpIYw2/KBvllkZmZSYGBgTRs2DC6c+cO237i/rpmzRqyt7en5ORkImqfVzMyMsjGxobef/99unnz5p+a8TU0NFBlZSVdunSJYmJiaO3atawphFhQ7EnrV3JyMk2bNo38/PwoICCAgoODWd+NzMxMsre3p/Xr15NQKGTrffjwYTI1NZWYK8WboO+//56cnJwoOjqaiNrlg7t371JaWhrt3buXNmzYQH5+fmRnZ0fbt2/vlhv0wsJC+uCDD+jAgQPU2NhIH3/8MXl7e3cwXcrOziZHR0dWThLPe/Hx8RJKCDHiNjp9+jQxDEPHjx9n712/fp3Wrl1LW7ZskXD+705woQZfAvX19Rg5ciRiY2ORkZEBdXV1DBo0SCKLUnFxMU6dOoUZM2ZATk4Oz549Q3JyMkQiEVxdXdkQYtLS0mwqaVVVVQgEAsTExEBPTw+mpqYA2mMER0ZGwt7evsenTE5PT4eioiLq6+tRUFCAyZMnY+bMmWy4oPDwcBgYGMDGxgY8Hk8ifJCMjAwyMjJQWlqKUaNGQUtLC0KhECdOnEB5eTmsra3ZY8/6+nrcvn0boaGhEAgE+Pjjj6GmpgZ5eXlYWFhAU1Ozi1vi1UJE2LRpEy5fvozPPvsMkyZNgoWFBbS1taGsrMzG7BWHY9TQ0EBUVBR4PB68vb079EFZWVlkZWWhtLQUY8eOBRHh6dOnCAgIQHp6OrKystDQ0IBPPvkEn332GXtU2l3p1asXnjx5gsLCQgQHB0NLS0uizlVVVVBUVASPx8PZs2cRGBiITZs2wc3NDerq6h2OjHk8HoqKinD16lUEBwd3SPtMRJCXl4e1tXWPD7H2ZygrK6O6uhrHjx/H8OHD2bbU1NSEoaEhjIyM2DB1UlJSb3QisX/CgQMHkJSUhGfPniE7OxsCgQA2NjZsnxs0aBD27NkDBQUFODo6Ql5eHioqKsjPz0dFRQXefvtt9O7dG8Dvx/1/HBdr167Fhg0bkJubi4KCAowdOxY//PADm4mwJ6xfMTExWLBgAX799VcYGhrC0tISNTU1yMzMxJMnTzB48GDo6OjgwYMHSEhIQFBQEOTl5XH8+HGEhYVBJBJh7NixbKhQsWle//79ER0djfr6evj4+EBBQQHnzp3DunXrkJOTg/v372PIkCEIDQ2Fl5cXGz63O3D9+nV8/vnn2LBhA6SlpTF06FCYmZlBVVUVBw8ehLGxMczMzNh1++HDh4iMjES/fv3g7+/P9kFjY2McO3YMjx49gr29PdTV1SEUCtn7ubm5SEpKwvz589GnTx8A7fkCBg8eDFdXV+jq6nZZG/wZnPD9EpCTk8PTp09x/fp1SEtL49y5c2hoaICrqyuAdmG5ra0NiYmJkJKSwuDBgyErK4uSkhKcO3cONjY20NfXZxeR5xeQxsZGHDt2DAKBAAEBAQDag8K7urpi6NChXVHdTiM7OxtTp07Fw4cPMWnSJDg5OcHOzo7dnBgaGiI+Ph4lJSVwdHRE7969QURs+2lra6OoqAiJiYnw8/ODtrY2zM3N2TZNT0/Hw4cPceXKFZw7dw47duxAZWUlFixYACcnp66seqdz/fp1fPPNN1i8eDHeffddKCgoAPg9a6J4gWxqaoKsrCx0dXVx6dIl5OXlsamOhUIhm7pXSUkJp06dQlVVFcaPHw8ZGRmoqKjAysoKTk5OeO+997Bw4UJ2Q9ndkZaWhpKSEqKjo6GsrMz2j0uXLmHx4sU4efIk3nnnHVhbW2PPnj3Q0NCAr68vlJSUQO0njGwbivtnSUkJYmJiMGbMGKirq0s80xMEkpeBnJwcFBQUEB8fD2lpabi5uUm0jbjNxAs4x19DvM6oqamhqKgIdXV1MDMzQ0REBJv6XU5ODsrKyigpKUFycjLs7Oygr68PeXl58Pl8HDhwAMbGxjAxMYG0tLRE2u7q6mq0trZCQ0MD/fv3h5OTE/z8/LB8+XK4urp2KyHx35Keno758+dDTU0N69evx7vvvouRI0fC19cXT58+RUJCAuzt7WFiYgI5OTmcOHECeXl52LJlC2JiYqCtrQ2gPWleXl4e3N3d2fjbqqqquH//Pi5fvgwDAwOYmJigX79+8PT0hK+vLxYsWABvb2/w+fyubAIJkpOT8dVXX2Hr1q0wNDRESEgI5syZwyoSevfujdu3b+PKlSvw8vJCr169QEQ4duwYioqKEBwczG5CWltbIS0tDRUVFRw+fBitra3w9vYGj8dDfX09Ll++jPXr18PW1hbvvvuuRDzw7q604ITvv0leXh7Kysqgra3NJgcAAF1dXRw6dAijRo2Cqqoq4uLicP/+fTbovYqKCgoLC5GQkIDZs2dDRUUFioqKSEtLQ15eHhiGgY6OjkQWquzsbOzZswd8Ph+rV69mhUsA3c4J7WUi1q4qKyujsrIScXFxePfdd9G7d282UYlYu83n8xEREQGGYWBhYcFO/kTETvDilOfDhg2DnJwcBg8eDGNjY+Tn5+PKlSsoKSlBaWkpAgMDsXHjRpiZmXVl9V8pRUVFuHTpEgQCATQ0NNj2OnToEK5cuYIvvvgC6urq7G8gFmpu3LiBb7/9FpmZmWAYBqqqqlBWVsbJkychFArZCVH8vqqqKmzevBmurq4YPnw4gHaB0tDQEIMGDWIXnNcJdXV1FBQUICUlBUpKSvjqq68QFhaGQYMG4YMPPkC/fv0gKyuLhoYGnDx5Eubm5jAxMQHwu8AtEAhQXl4ORUVFXL9+HWlpafDw8MCAAQM44fEPiAVrRUVFPHz4EImJiZgxY4bEM1yb/W9qa2tx48YN1NXVoba2Furq6mx/1NLSQlFREXJycjBnzhwYGBhg+/btuH//PqysrKCqqgpDQ0Ps2bMH6urqcHBwgKysLNTU1JCdnY3bt2/Dx8eHzU5ZUVGBixcvIjQ0FEVFRbC2toauri5MTExgYGDQxS3xchH3T1lZWZSWlqKqqgrz5s1jT/EUFRXR1taG2NhYMAwDe3t7KCgooKqqCvHx8fD19cXKlSvZ09za2locP34cly9fhp6eHqux1dXVxZ49e1BfX4/Ro0eDz+dDU1MTOjo63UrAjI6ORkhICA4fPoympiYAQEhICAICAqCqqsqWVV5eHgoKCoiIiICZmRkKCgrw2WefIT4+Hk1NTWhra4Oamhr09fXZDZ2ZmRkKCwtx/Phx3L59G7du3UJqaip27doFeXl5LF68GP379+/C2v99OOH7LyIQCPDLL79gwYIFuHDhAkaOHAkVFRUA7cJi7969UVBQgNzcXAQHB8PJyQk7duzA5cuXMXjwYBgaGkIkEuH06dNQVVWFjY0N+vbtCw0NDRw8eBBpaWmQkZFBW1sbbty4gaSkJOzYsQPV1dWYO3cunJycWK1uT1twmpqa2Cyc4uN8kUgEeXl5EBHOnj2LxsZGeHp6ssdx4sXD1NQUUVFRqKyshJOTE1RUVCSOP7W0tHDnzh0kJSVhwoQJ4PP5kJKSgqmpKcaOHYv33nsPXl5eCAkJgYuLS4/SyDxPWloavvzyS+zduxeJiYkoLy+Hra0t24eTkpJQWFiIBQsWQE5Oju1jJSUlmDNnDrZt24aysjLcuXMHrq6u7JF/Tk4OkpKSoKenh0GDBqG+vh55eXnYvHkznj59is8++4zNJva6IysrCwUFBcTFxSEuLg7Ozs5YtmwZpkyZAlNTU7bvWFlZ4eeff4aMjAyGDBkCOTk5NDc348GDB4iNjUVoaChqamrwzjvvwN/fH87Ozl1cs+6JuM8oKiqioqICcXFxcHNzQ9++fbu4ZK8HRUVFWLt2LTZv3oxz585hz549iIiIwPXr12FkZMRugOXl5ZGeno6qqir85z//gYaGBsLDw3Hx4kVYWFjA3Nwcd+7cQXp6OhwdHdG3b1/IyclBVlYWBw4cgLm5Ofh8Ps6fP4/Nmzdjz549kJKSwowZM177jKvPU1NTg7y8PFYxJO6fKioqEIlEOHLkCIyMjGBmZsaeLFy9ehUpKSlwd3eHvb09lJWVIRAIkJSUhHHjxsHX1xeKioqQkpKCs7MzBg0ahMjISKSmpsLc3ByamprQ0tJCQ0MD/P39O0ST6S58/vnn2L59O3x8fNjTjaysLFRVVWH06NESp3pAuyLjxo0b2LdvH9LT0+Hm5obZs2djyJAhuHjxIsLCwiAvLw8jIyP2FMDR0RFaWlpIS0tDUVERHj16hPHjx+OHH36AoaFhV1X9n9PJNuavNStWrGBjzU6dOpUyMzOJ6HfDf3EQd3ECltOnT5Ovry95eXlRTEwMVVZW0nvvvdfBgfLYsWMUFBREZmZmZGNjQ25ubuTk5ESrVq36r17OPYW6ujry8PAghmFo2LBhdOrUKYn71dXVtGDBArKwsGCdSf6YCXHv3r1kZmZGJ06ckPis+LlTp06Rs7Mzm1HtTUqM09zcTBs2bCBnZ2eaPXs2RURE0JkzZ6iyslIissGKFSuIYRjWCUp8r7S0lJYsWUIpKSmUl5dHrq6utGzZMjZKQXZ2Nk2fPp0YhmEdjNzd3cnJyUkiIUpPoba2lubPn0/e3t4SccnFiPvW8uXLyd7enhISEqi4uJi2bdtGw4cPJ4ZhaMGCBd02dnl3ZcOGDWRra9shGQlHR3JyctgxOW7cONq6dStFRkZSYmIirVy5kszNzcnPz4/S0tKIqH2eXL16NZmZmbFZko8fP04BAQHk7u5OJ0+epIKCAmIYhrZu3co6wj158oQmTpxI3t7ebJbFqVOnsg7uPYXCwkJasGABOTo6kr29Pbm4uNDOnTslkuM9fPiQJk2aROPHj2fnvJs3b9KYMWPIxcWFDaJA1B48YNq0aTRmzBiqq6vr8H1paWnk4+NDDMOwGSu7O1VVVZSRkcEmViMiWrJkCTk4OLBj9vm1QCgU0vHjx9lIWg0NDey98vJyWr58OZmamlJISMgLo+j8MWLM6win+f4LiM1LCgoKcPfuXWzduhWJiYlITEzE8OHDWe2htrY2bt26hcuXL8PJyQmurq5wd3fHtWvXEBERAQ0NDfTr1w/nz5+HgYEBGIYBEcHMzAz+/v7w8fGBj48Phg4diq+//ho+Pj7srq+n8nfs5aWlpeHs7Mz+HuKdtJmZGX799Vc0NzfDycmJtbMF2rVnffr0QUZGBhISEjBz5swOzm09EXEbZWZmYvPmzZgwYQJCQkLg5uYGY2Nj1jlQbF5CRIiNjYWmpiaGDBkCHo8HIoKKigo8PT0xcOBA9OnTB0+fPkVMTAwGDx4MPT096OjoYMSIEez3Njc3w8/PD1u2bIG9vX2Pc4CTl5eHSCRCVFQULC0tMWjQoA7PiI9Jf/75Z+Tn5yMyMhJpaWnw8/PDrl27MH78eHbO4PhziAgnTpzAgQMHYGZmhunTp3ero/buxq1bt/Dhhx/i0aNHWLNmDWbMmAEPDw/Y2NjAyMgIXl5e0NfXR0xMDO7fvw87Ozuoq6tDSkoKFy9exKNHjzB69GiYmJjA19cXGRkZOHDgAAYOHIh79+6hvLwc5ubm0NbWhpycHFpaWhAXF4fBgwdj8+bNmDFjRo85mcjLy8Py5cuxZs0ayMvL45133oGtrS0qKytx4sQJKCsrw9HREQDYNefQoUNQUFDA7t27sW7dOujr62PTpk0SGmtxjPNDhw5h0KBB7BxC/68dNjAwgI+PD8aOHYvBgwd3fsX/AYqKitDX14eCggJbD2lpaaSlpaG6uhojRoyQ8AmQkpJC7969ceHCBdTV1bHOpEKhEMrKynB3dwcRISoqCklJSbCwsICOjg77brHf12tNV0n9ryNJSUlkY2NDVVVV9Ntvv5Gfnx9NnjxZQhtz8eJFYhiGwsLCWA1BVVUVffrpp+Ts7EyjR48md3d3GjdunMS734QQWbm5uWy83ud3wcXFxWRpaUlfffUVzZo1i2xtbWnJkiVsCLZHjx7Rhx9+SI6Ojh3eKW63zZs3k7W1NSUkJEjcF2u/09PTe3y4wJaWFjYklZiQkBDy8fGRiDFP9LuGVtx+LS0t5O3tTf7+/mw7vUhrXVlZSdbW1rRu3To2w6WYnhQy7M+oqqqioKAgeueddyQ0NmLE7TZz5kxiGIa2bdv2RocL/LeEh4fTrFmzJLSHHC+moqKCli5dSu7u7h1OV54/8du4cSMxDEOhoaFE1B4vesmSJWRpaUl3795lnystLaXVq1eTs7Mz2dvbk7OzM+3YsYMd69XV1VRbW9sJNet8goOD2fTmNTU17Hz36NEjsrGxoREjRkhoru/evUtTpkwhhmFo0qRJFBsby4YZ/OP6XlJSQkFBQTR58uQXanZf17lUJBKxZW9paaEFCxaQs7Mzm73z+XYQCoVsyOXn45M//8zevXtp0qRJlJ+f30k16Dw4zfffIDc3F9euXYODgwOsra3h6OiIqKgopKenw9fXF3w+H3p6erhw4QLy8vLg7OwMdXV1KCoqwt3dHerq6jh58iSamppQXl4OOzs71gmlJ9jE/jdepr187969YW1tzdrUiduNYRjs27cPMjIycHBwkNiBA4CBgUGPDxcYGhqKdevWwdbWlu1XFy9eRElJCSZMmMBqXJqbmyEnJ4fW1la0trZCVlYW0tLSICLExMRAIBBg2LBhrPZb3IYCgQDx8fFISEiAq6srXF1dO/TbntyPxSgqKkIgEODQoUOwtbXFgAEDJO7T//tmeHt7Y/HixXBycuK0tf8Ca2trjBkzpmdou14itbW1UFBQYE+vAIDP50NaWhrHjh2DpqYm7Ozs2OefP4WysbHB3r17UV1dDS8vL/Tu3RsikQjp6emora2Fj48PhEIhVFVV4eXlBVlZWVy/fh21tbVQU1PD4MGDoaioCAUFBcjLy3d63V8F8fHxyMnJgbm5OYD2cHVnz56Fvr4+Ro4cyfpkqampIScnB/n5+fDy8oKOjg6Ado12Y2Mj0tPTsWzZMvj5+bFRo/44LyopKaGpqQmHDh2Cg4NDB5vl13UeFWu3RSIRZGRkQERITU1FU1MTvLy8OkR00tDQQGJiIurq6uDi4iLRl6SkpGBra4vx48ezIQR7Epzw/Reg545Rdu/ejYkTJ6Jv377o06cPzMzMkJiYiKysLBgbG0NTUxNKSkqIiIiAiYkJLC0tWY9oKysr6Ovr4/Hjx1BSUsK4ceN6ZKf6IzweDydOnEBDQwMaGxtx9epV6OrqQldXl41aoquri59++gn6+vqYPn06jIyMkJaWxi4iQ4YMwfXr15GcnIzp06dLLCQikQh8Ph+PHj1CVFQUXF1dYWBg8NpOYH8X8eIrKyuLq1ev4u7duxg7dixEIhFaWlpw7NgxlJWVQUpKCmfPnkVsbCwiIiIQGhqKa9eu4dmzZ7CysoKtrS3Onz+PlJQUqKqqon///lBQUEBjYyMePnyI06dP4/Dhw3B2dsbixYs7LLpvSnsD7VEiMjIycO/ePfj5+UkI1+K++bzjKsc/h2tDSaKjozFr1iyUlpbC09OT7XvidUpJSQl3797F+fPnERAQ0MF0USgUQl5eHuXl5UhPT4ednR0GDhzYwTld7BzI4/Fga2sLU1NT2NnZ4bPPPutx5pAVFRVYs2YNoqOjMXPmTPB4PPTr1w+XLl1Cbm4uzM3N2QhjjY2NiIyMRK9evfDBBx+wih4ZGRkoKCggIyMDlZWV8PPz+699l8fjoVevXjA1NcXw4cN75OZcSkoKffv2RU5ODi5duoTRo0ezzqnidlFWVkZzczMOHDiAwYMHs+v2mzDmOeH7LyDuCI2NjTh16hSMjY1haWmJ+vp6ZGZmIjk5GSUlJWzYJWtrayQkJODBgwdwcnJCr169WBtcY2NjjBo1CjNmzHgjBO9XaS//R+23ubk5+vTpg8DAwB5na/w8hYWFuHTpEh4/fozHjx+jX79+4PF40NbWRnFxMc6cOQNPT0/o6OhAS0sLLS0tOHLkCOLi4nDx4kXk5uayWnDxYtuvXz+YmZnB2NgYlZWV2L9/PxISEnDt2jWkpaXhwIEDOHXqFFxdXfHxxx+/EX33z+Dz+WhoaMDBgwfh7OwMfX39Ds+8CQsIR+cTGhqK3NxcVFVV4f79+3B3d2dPrsTCN4/Hw9GjR2FgYAALCwuJz4vnRqFQiJiYGHh6esLMzAzy8vIQCoW4cOECWlpa4O7uzipHgPbTQysrq06vb2egpKQEaWlpJCUlQVtbm20zZWVlHD9+HEpKSnB3d0dDQwN27dqFs2fPwsbGBp6enuDz+Wzbq6iooL6+HkeOHIGHh8efhlTt3bs3LCwseqzgLY5Y1tzcjNTUVACAq6urRJ/i8XhQVVVFcnIyXF1du200l1cBJ3z/DWpqanDmzBmoq6sjLy8Py5YtQ3x8PLy9veHp6YnLly/j2rVrcHFxgYGBAXbv3g0jIyNYWVlJdLaeckz3VxDXu6amBocPH8a8efPg4uKC06dPIz09HQMHDmSP7fr06YPw8HDo6+vD0tISmpqa8Pb2RllZGaKiovDo0SNISUmhoKAAQUFBHQRsZWXlHunkJ+bcuXP48ssvER4ejkuXLiEyMhLHjx9HUlIS9PX10b9/f8jLyyM1NRVlZWUYOXIkFBQU4OHhAVdXVzg7O2P48OH48ssv8e6772Ly5MlwcnJCSkoKbt68iSlTpqBv374YPnw4dHV1UVFRgcrKStTW1sLc3Bzr16/HxIkToaqq2tVN0eVISUlBU1MTRIShQ4dybcLxyhErMkpKSvDgwQOsXr0a69evR1NTExwdHSErK8s+o6ysjPz8fFy+fBmBgYEvdDK/e/cuTp8+DTs7O9jb2wNAj3dOr6ysxK1bt6Ctrc2uE2LBWVlZGbdv30Z6ejqmTJkCADAyMkJKSgpyc3ORk5ODlStXoqCgABoaGrh37x6ysrKgq6sLPT09AICMjAwUFRVx/vx51NXVvTD775uElJQUdHR0cPXqVVy9ehVvv/12B5NQdXV1TJ06lc14+sbQFYbmrzP+/v7EMAy5uLjQ559/Tunp6VRXV0etra2UkpJCpqamFBQURHfu3KFx48bRkSNHXlvniZfJqVOnyMvLiw3PmJeXRyNGjKCxY8dSRUUF+9y7775L48aNo4KCAvZafX09HTx4kCwsLMjc3JwYhqH09PROr0NXcfLkSRo+fDiZm5vTRx99RAcPHqTk5GS6ffs2hYaGkqOjI3l6elJKSgoJhUL64osvyNLSknVQa21t7fDO551aFi1aRAzDvNAhtaWl5YUOQRwcHK+OO3fu0LFjxygjI4OIJJ2fd+/eTcOHD6eWlhYKCwsjDw8PWrVqlcTnhUIhRUVFEcMwFBsby14XiUTs2I+MjCSGYSRCDhL1bOf0CRMm0NChQ9n15Y9r8/79+4lhGIqLi2OvnThxghiGIWdnZ9q3bx8VFRVReXk5ZWZm0rBhw8jW1pYiIiLYeba2tpbWrl1LDMP0SEfBv4O4fcPDw8nR0ZG2bdtGRG9GgIn/Baf5/hs0Nzfj5s2baG5uxq5duzBq1Cg2Zay0tDT69+8PkUiEixcvYsiQIVi8eLFE1sU3EeLs5f8VeXl5CAkJAY/Hw5YtWxAUFAQHBwcMGDAAmpqaGDx4MExMTHD69Gnk5eXB29sb2traSEhIYDUvgORRMxFJHHUeO3aMTfoizs4mRlpaWiJlLwcHx6sjLS0NX3zxBfbv348rV67g/PnzmDBhAptwTEpKCrdu3cLVq1cxevRoODo6onfv3vjxxx9BRLCwsGB9DZSUlJCdnY3c3FyMHDkSsrKyrD1tQUEB1qxZAzU1NcyZM4dN9AL0LOf0goICpKWloaqqCvr6+lBWVsaJEydgYmLSISOylJQU+Hw+srOzce3aNQQFBQFod+ZPTEwEn8/H7NmzYWRkBCUlJejq6sLNzQ1lZWXYt28fGhoa4OzsDD6fz4Yk9fb2hrKyclc2QZcibld1dXUkJiYCAEaOHPlGy0QsXSn5v24IBAKaO3cuTZgwoUPyG7Fmggsp9mIePHhArq6udPDgQSJqT64TGRlJgwcPJhsbGwoKCmITtwQGBtLkyZPZQPriEFkCgUAiscGbQEtLC61YsYJcXV0lQoeJRCIJbdi2bduIYRj68ccfSSgUUkhICNnZ2XUIMfj858vKymj37t1ka2tL69evf+V14eDgeDECgYDWr19P7u7uFBwcTBEREZSSkkIPHz5knxFrEXNycohhGIlEIxs3biRvb2/67rvv2GstLS20b98+YhiG1aDX1dVRWloaffjhhxQYGEhXr17tpBp2LqmpqRQUFETu7u7k7e1Nvr6+1NDQQEKhkAICAiTWl+e130KhkHbu3CnRZkTtmls7OzvavXs3EXVMGPPJJ58QwzC0aNEiTgb4L3CJxSThNN9/Ax6Ph2vXruG3337DmDFjoKSkxO7gxJrFnug88TLg7OX/GdLS0lBSUkJUVBRUVVXZpA5SUlIdQoeFh4ejrKwMY8aMAZ/PR0JCAkQiEdzc3NDW1oa6ujrcvXsXly5dwsWLFxEeHo7o6Gh4enoiODiYs1vm4Ogirly5gu+//x6BgYFYuHAh3Nzc0L9/f6iqqrLaQ/Fa8+DBA6SmpmLgwIFgGAZAuwPwuXPnkJqaCmDp6AsAAAgGSURBVHl5eZiYmIDP50NBQQFZWVkoLCyEpqYm9u3bh59++glEhI8//hhubm4Aeo5zsFAoxMaNG7Ft2zYYGBjg7bffxpgxYzB9+nRoaGiwJ6n/1969hUS9RXEc/+Y0pmaoZaXZiDEWTmLmBZS0MofxQJKaFPggll2gIpMuENFDQhBlGAUJQdJFs7EbRA8ZQViJUVFQkT3UQ9BUZomUmpk6dh7COWfonNP1zGj+Po8zImv/Gf7/tfbe/71qa2uJjY0lJibGbfbbx8cHPz8/bt26xZMnT8jNzQVgxowZXLhwgVevXpGenu46tWNgYACDwUBycjIJCQkUFBSM6Jnu/6LGYu5GezuA4cZkMuFwOIDf54blCZGRkfj4+HD06FFCQkKYN28eOTk5xMfHM2bMGBITE1mzZg2lpaXs3LmTmTNnYjQa3V7MGKni4uLIyMjg9OnTLFmy5IvtNk6nE19fX/Lz87Hb7Tx8+JDU1FSSkpI4d+4c69evx8/Pj5cvX7J161Zev35NYGAgU6dO5eDBg8ydO9dLIxMRgOvXr+N0Otm+fbvb5/39/Ywe/fkx/elvS/iDJz09ePCAY8eOcfnyZaZNm0ZCQgLHjx+nvb2drVu3Mm3aNHJzc9m3bx+NjY2Ehoaydu1aioqKvDHM/93du3epq6tj8eLFFBcXEx4e/sXWkuzsbA4dOkR9fT2pqalMnjzZ7Tkzffp0rFYrR44cweFwYDKZ8Pf3Jzc3l+rqahoaGigoKAD+mmybMGECNpvNO4OWYUkz39/p9evX9Pb2YrPZ8PPzG/GJ4bfSfvkfZzQa8ff358yZM4SHhzNr1iy37wdnwHt6erh48SJpaWnMmjWLrq4url27hp+fH4mJiYwdO5aUlBTS09MpKipi+fLlXzR3EBHPGzyuds6cOa7i+sOHDxgMBgYGBlyNseDzqU4nT57kypUr2O12+vv7KS4uprS0lOzsbDo6OqipqSEwMBCLxUJwcDCdnZ2sXr2aPXv2EB8f782h/q/sdjv379+ntraWcePGuTUIGzyO0dfXF6fT6WqSZTab3RL00aNH8+7dOxoaGoiIiHBdr+joaI4ePUpLSwvZ2dkjbhVWfi3NfH8nm81GVlaWt8MYdoxGIz09PQQHBzNp0iS3LQ5OpxODwcCaNWvYsGGDF6McupKTk0lNTcVut7No0SK36zc4a9Pb2wtAe3s7APPnz+fChQscOHCAVatWERAQQExMDDExMV4Zg4j8s4KCAs6ePUtFRQWFhYU0Nzfz/PlzXrx4waNHj5gxYwaLFy8mLy+P9+/fExkZSUtLC9u2bSMpKYnw8HDXi9ElJSU8f/6c3bt3ExERgc1mo6Kiwssj9IyPHz+6jlkcvM99+PABo9FIf38/vb29BAQEkJ+fz+HDh7l06RKJiYmEhobidDqBz7PZsbGx9Pb2MmHCBOBz8j5+/Hg2bNhAVFSUtpbIT1Py/Z00E/tjDAYDYWFhPH78mI8fP7ot8w0u3f1uZ8r+SoGBgeTl5bFlyxauXr1KTk4OgNt1bG1tBSApKQmAqVOnsnDhQlJSUlxt5PX7FRl6zGYzK1eupLKyksbGRuDzipbFYiEqKopHjx5x7949Ojo6WLFiBd3d3ZjNZhYsWEBwcLDr/wwMDBAUFMTevXt58eLFiCu0v6WIycvLY+nSpRQVFVFZWUlGRgY5OTmu51B3dzdnzpwBcHW1HPyuuLjYOwOT346Sb/EY7Zf/OWlpacyePZva2lqsVqvbC79NTU1UVVWRnJyM2Wx27RVdtmyZl6MWkW+xdu1arFYrra2tdHV1kZKS4mrj7nA42LRpE6dOncJmsxEbG8v9+/cJDg52rRzCX1vQAgMDR1ziDV8vYpqbm7l37x6dnZ0sX76curo69u/fT1hYGGazmZaWFpqamjh//jyFhYXMnj0b0PNKfj0l3+IxU6ZMwWq14uvrqxcpf0BISAh5eXmUlZVx9+5d5s2bR1tbGzdv3qSmpobo6GjKysoICAjwdqgi8p0MBgMWiwWLxeL6bDCxtlgs2Gw2Tpw4QVtbG0FBQTx58oS2trZ/7Hcwku+tXytiNm7cSF1dHTk5OezatYvy8nKKiooIDQ3FaDTy5s0bli5dyrp167w8EvmdKfkWj9F++Z+XmZmJ3W6nqqqKrq4uzp8/z61bt0hISGDTpk2YzWYVNiLDXH9/v6s52aCnT5/S3d2NyWSiq6uL8ePH8+7duxHRbOx7fK2IycrKorq6mlevXpGWlsaxY8doaGjA4XDg7+/PkiVLvmg2JvKrKfkWj1FC+PMmTpxIbm4u5eXl3L59m7i4OCorK0lPT3f9ja6zyPD29+MFW1tbqa+v586dOxQWFhIaGkpJSQk7duzwcpRD378VMT09Pa7Z8JCQEPLz870VooxQSr5FhpFRo0aRmZlJR0cHVquVuLg4b4ckIr9QZ2cnjx8/xuFw8PbtW27cuMGNGzdYsGABhYWFwOeEUb7u34qYZcuWYTabvRydjGSjPn369MnbQYiIiAj09fXxxx9/0NfXx5gxYzCZTKxcudJtdUu+7r+KmM2bNxMVFeXtEGUEU/ItIiIyhDx79oyWlhamTJmCyWTydjjDkooYGcqUfIuIiMhvR0WMDFVKvkVEREREPMTH2wGIiIiIiIwUSr5FRERERDxEybeIiIiIiIco+RYRERER8RAl3yIiIiIiHqLkW0RERETEQ5R8i4iIiIh4iJJvEREREREPUfItIiIiIuIhfwK45VMrSaPrLwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d9vdM7GMoCVy",
        "colab_type": "text"
      },
      "source": [
        "## Load model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mARr6dl4oDiq",
        "colab_type": "code",
        "outputId": "75791513-e4af-4410-aeec-4c79d95000fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 639
        }
      },
      "source": [
        "model = BertForSequenceClassification.from_pretrained(bert_model, num_labels=NUM_CLASSES, output_attentions=False, output_hidden_states=False)\n",
        "model.cuda() # Tell pytorch to run this model on the GPU.\n",
        "\n",
        "print_model_params(model)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                           (8, 768)\n",
            "classifier.bias                                                 (8,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9KNNF1iFTChI",
        "colab_type": "text"
      },
      "source": [
        "## Load built-in tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wwQo6cOdW8Iy",
        "colab_type": "code",
        "outputId": "23159bdd-95cb-4df0-8b94-d0d2d608bd94",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "source": [
        "bert_model = models[BERT_MODEL][0]\n",
        "lowercase = models[BERT_MODEL][1]\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained(bert_model, do_lower_case=lowercase)\n",
        "print('Text will be split into tokens using the', bert_model,'built-in tokenizer.\\n')\n",
        "\n",
        "print(\"Hello I am a MONKEY \\U0001F648\")\n",
        "print(tokenizer.tokenize(\"Hello I am a monkey \\U0001F648\"))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Text will be split into tokens using the bert-base-uncased built-in tokenizer.\n",
            "\n",
            "Hello I am a MONKEY 🙈\n",
            "['hello', 'i', 'am', 'a', 'monkey', '[UNK]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gHavoz08cbL0",
        "colab_type": "text"
      },
      "source": [
        "## Analysis of tokenized sentences length"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CcthKY7SceoY",
        "colab_type": "code",
        "outputId": "b955683f-de21-4d33-8f91-e7fda21bf8c8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 407
        }
      },
      "source": [
        "lengths = []\n",
        "for sent in X_train:\n",
        "  input_ids = tokenizer.encode(sent, add_special_tokens=True)\n",
        "  l = min(MAX_LEN, len(input_ids))\n",
        "  lengths.append(l)\n",
        "\n",
        "plot_lengths_distribution(lengths, title=\"Length distribution of tokenized train sentences\")\n",
        "print('Increase MAX_LEN if too many truncated sentences, decrease it if too much padding.')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.1% of the training examples (11 of them) have more than 48 tokens\n",
            "Increase MAX_LEN if too many truncated sentences, decrease it if too much padding.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo0AAAFjCAYAAABYCzEiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeVhV5f7//xcgoDIoGjjgbLJNkUErNa0cSNGjCZppKmqapplfh/PJoZPndLKOZaRZWIrZIJlpDpGaI3UaNKmcSCUN0tQU2U4IKJPs3x/+2MfdRjco05bn47q4invd697vxW35Yt1rcDCZTCYBAAAAN+FY3gUAAACg4iM0AgAAwCZCIwAAAGwiNAIAAMAmQiMAAABsIjQCAADAJkIjAFwnIiJC3bp1K+8yiq1bt26KiIgolbHj4+NlMBi0du3am7aVtopSR3l+LlCeqpR3AQAKd+LECUVHR+unn37S6dOn5eLiorvuuksBAQEKDw9Xhw4dyrtElLEPP/xQnp6e6t+/f3mXclvs5TgSExO1fft2hYeHq0GDBuVdDlDuCI1ABfTLL78oIiJCVapUUVhYmO6++25lZWXpjz/+0I4dO+Tm5kZorISWLVsmX1/fChG27rvvPiUkJKhKleL/NXKrx3E7n3krEhMTFRUVpfvvv98qNJZ1LUBFwJ92oAJauHChrly5otjYWLVs2dJqu9FoLIeqgP9xdHSUq6trmXxWRkaG3N3dy/QzbalItQBlhWsagQro2LFjqlmzZqGBUZK8vb2t2nbu3KlRo0bp3nvvVZs2bdS3b1+tWLGi0P1XrVql0NBQ+fv765FHHtGHH36oNWvWyGAwKD4+3txvxowZMhgMhY5hMBg0Y8YMq/Yvv/xSTzzxhIKDgxUYGKiBAwdq8+bNN9x/7969GjZsmIKCgtS+fXv94x//UGZmplV/o9Gol19+Wd27d5e/v786duyoJ598Ujt27LDod+zYMT333HPq3Lmz/P391a1bN7322mu6fPlyocdRVEUdt+Bnlp6ern/961/q2LGj2rRpo8GDB2v//v1W4164cEEzZ85U+/btFRwcrOHDh+vQoUNW11YaDAb9+eef+vHHH2UwGMxfJ0+etBgvOTlZY8eOVXBwsNq1a6f/9//+X7F+ydi+fbvCwsLUpk0bPfzww3rzzTeVl5dn1a+wa/ry8/P14Ycfqm/fvgoODlbbtm3Vs2dPPf/888rNzS3ScRRcm3no0CGNHj1a7dq106OPPnrDz7xeTEyMevbsqTZt2qhnz56KiYmx6nOjaz//Ovbbb7+tmTNnSpKGDx9urrPgz/yNarl8+bLeeOMNhYSEyN/fX506ddK0adP0559/3vDz1qxZo7/97W/y9/dX165dtWTJkkKPDyhvnGkEKqBGjRrp6NGj2rp1q3r06GGz/8qVK/Wvf/1LQUFBGjdunKpVq6adO3fqxRdf1PHjxzV9+nRz3w8//FBz5sxRy5YtNXXqVF25ckXvv/++ateufdt1z58/X4sWLdKDDz6oSZMmydHRUdu2bdOkSZP0z3/+U0OHDrXon5iYqHHjxql///7q06ePfvzxR61evVqOjo6aPXu2ud/Jkyf1xBNP6Ny5c+rXr5/8/f115coV7d+/Xzt37lSnTp0kSQcOHNCIESPk6empQYMGqU6dOvr1118VExOjvXv3KiYmRs7OzsU+rlsZd/To0apVq5YmTJigixcv6oMPPtDYsWMVFxcnd3d3SVJOTo6efPJJJSYmqn///mrTpo0OHz6sJ598UjVq1LAYb+7cuZozZ468vLw0btw4c3utWrXM/37mzBkNHz5cISEhmjZtmn799VetXLlSGRkZev/9920e57Zt2zRx4kT5+vpqwoQJcnJy0tq1a/XNN98U6ef07rvv6q233lLXrl01ePBgOTk56eTJk/rqq6+Uk5MjZ2fnIh3HqVOnNGLECIWGhqpHjx5FCvwff/yxjEajBg0aJHd3d23YsEEvv/yy0tLS9Oyzzxap/us98sgjMhqNWrlypcaNG6dmzZpJuvbf5o3k5uZq9OjR2rNnj3r27Kknn3xSf/zxh1asWKEdO3ZozZo1qlu3rsU+n376qc6ePavHHntMnp6e+uKLLxQZGam6deuqb9++xa4bKFUmABXOnj17TK1btzb5+fmZevToYZoxY4Zp+fLlpqSkJKu+Z86cMfn7+5umTp1qtW327Nmmli1bmo4fP24ymUymtLQ0U2BgoKlXr16my5cvm/udPn3aFBQUZPLz8zPt2rXL3D59+nSTn59foTX6+fmZpk+fbv7+wIEDJj8/P9Mbb7xh1Xf8+PGm4OBgU3p6usX+BoPBtG/fPou+Y8aMMbVq1cqUkZFhbnvqqadMfn5+pm+//dZq7KtXr5r/vW/fvqaePXtafI7JZDJt3brV5OfnZ1qzZk2hx3K9YcOGmbp27WrRVpxxC35m//rXvyz6fvnllyY/Pz/TihUrzG0ff/yxyc/Pz/TOO+9Y9C1o/2sdXbt2NQ0bNqzQurt27Wry8/Mzbdy40aL9xRdfNPn5+ZmSk5Nvetx5eXmmhx9+2HT//febzp07Z26/dOmSqUuXLlbHuWvXLqu2sLAwU69evW76OUU9jlWrVlltK+wzC9qCgoJMp0+fNrdnZ2ebBgwYYGrVqpVF+40+u7Cx16xZY/XfxM36r1y50uTn52d67bXXLPp+/fXXJj8/P9P//d//We3fqVMn06VLl8ztly9fNrVv3970+OOPF/rzAcoTy9NABRQcHKw1a9YoPDxc6enpWrt2rf7973+rd+/eGjp0qE6cOGHuu2XLFuXk5Oixxx7T+fPnLb66deum/Px87dy5U5L0/fff68qVKxo6dKiqVatmHqMkzmqsX79eDg4OCgsLK7SOzMxM7du3z2KfoKAgBQYGWrR16NBBeXl55uW8ixcv6rvvvtODDz6oBx980OpzHR2v/W/s8OHDOnz4sPr06aOcnByLz2/Xrp2qV69utZRdFLc67siRI62OS5L++OMPc9vXX38tJycnDR8+3KLvwIED5eHhUexafXx81Lt3b5ufW5iDBw/q9OnT6t+/v8VZPw8PDw0ePLhIn+/u7q4zZ87o559/LmbllmrWrFnsm2T69u1rcRbPxcVFI0eOVF5enr766qvbqqeotm3bJkdHRz399NMW7V26dNE999yjuLg45efnW2wbMGCAxVxXq1ZNQUFBOnbsWFmUDBQLy9NABWUwGPTqq69Kkv7880/99NNP+uyzz/Tzzz/rmWee0Zo1a+Ti4qLk5GRJ1iHlemfPnpUk83VjBUtt12vevPlt1ZucnCyTyaRevXrZrKNAw4YNrfrUrFlT0rWwKEnHjx+XyWRSq1atbH6+dO1atLfffrtIn18UtzruX4/Ny8tL0v+OS7o2Hz4+PnJzc7Po6+LiogYNGujSpUvFqrUoP88bKfhF5Hb+bEydOlUTJkzQ0KFD5ePjo/vvv19dunRRz5495eLiUqQxpGvH4eTkVOT+N6rx7rvvliSLX7JKU8F8/vXSgoJaEhMTdeHCBYtLQQp7lE/NmjVtzhdQHgiNgB3w9fWVr6+v+vXrpyFDhmjPnj1KSEjQvffeK5PJJEl67bXX5OPjU+j+hYWJonBwcCi0vbAbI0wmkxwcHLRkyZIb/oVf8Jd4gZsFg4LjKq5Ro0YVekZSkjw9PW9pzFsZ90bHdqvHVRSl8fMsjuDgYG3btk3ff/+94uPjFR8frw0bNujdd9/VJ598Yg6wtlx/FrysXL16tcw/U7r5nAEVDaERsCMODg4KDAzUnj17lJqaKklq0qSJpGtnsh544IGb7l9wVuP3339Xx44dLbYVnFG7XsEZk4sXL1r8hV/YmZsmTZrou+++U/369W/7rOX1GjVqJAcHByUmJt60X+PGjSVdW6629XMojtIaV7r2y8APP/ygzMxMi7ONubm5Onny5G2F3OIq+MXi999/t9pW2J+NG3Fzc1PPnj3Vs2dPSdLy5cv10ksvafXq1XrqqadKpthCFFZjUlKSJMtfmm50Fq+wP9M3+qXpRho2bKjvvvtOly5dspq75ORkubu7m884A/aIaxqBCmjHjh2Fns3LysoyXz9XEMx69eolFxcXvf3228rKyrLaJz09XTk5OZKkTp06qWrVqlq+fLmuXLli7pOSkqL169db7VsQSAuuiSzwwQcfWPUteCzKvHnzCj1rcytLw9K1v+Qfeughffvtt1Z1SP87g9aqVSv5+fnp008/LTQA5OXl3dKSX2mNK117/MvVq1e1bNkyi/ZVq1YpPT3dqr+bm1upLVu2bt1adevW1dq1a3X+/Hlze0ZGhj799NMijXH9ftePK0lpaWnmttI4jvXr1yslJcX8fU5Ojj788EM5OTmpa9eu5vYmTZro6NGjOnPmjEXf5cuXW41ZvXp1q9pvJiQkRPn5+YqOjrZo/+abb3To0CF169bNfA0uYI840whUQHPmzNHFixfVrVs3+fn5qWrVquZgd+zYMYWFhZmfn1i3bl29+OKLeuGFF9S7d289+uij8vX11fnz53XkyBFt375dGzduVIMGDVSjRg1NmjRJr732mgYPHqywsDBduXJFn376qZo0aaJDhw5Z1NGnTx/Nnz9f//znP/X777+rZs2a+u6773ThwgWrmgMCAjRx4kS9/fbbCgsLU8+ePVWnTh2lpqbq4MGD+vbbb3XgwIFb+nnMmjVLhw4d0pgxYxQWFqbWrVsrOztb+/fvl6+vr5577jk5ODho7ty5GjFihB599FENGDDA4k0627Zt09SpU4t9g0VpjStdu+Hl008/1Ztvvqnjx4+bH7mzefNmNW7c2OoXh8DAQK1evVpvvvmmmjdvLkdHR3Xt2tUcbm6Hk5OTZs6cqcmTJ2vgwIF6/PHH5eTkpDVr1qhmzZo6deqUzTF69+6toKAgBQQEyMfHR0ajUatWrZKzs7P+9re/lepxNG3aVAMHDtTgwYPl5uamDRs26JdfftEzzzyjevXqmfsNHTpUGzdu1MiRIzV48GDl5uYqNja20CXxNm3ayNHRUYsWLVJaWpqqV6+uBg0aWN28VSA8PFzr1q3TkiVL9Oeff+ree+/V8ePH9cknn+iuu+7S1KlTb/n4gIqA0AhUQDNmzFBcXJx2796tLVu2KD09XR4eHvLz89OYMWOsAsqAAQPUpEkTvf/++1q5cqXS09NVs2ZNNW3aVJMmTbJ4GPioUaNUvXp1ffDBB3rjjTdUr149jRo1Sh4eHnr++ectxnV3d1d0dLTmzJmjxYsXq3r16urRo4def/113XfffVZ1P/vss/L391dMTIyWLVumy5cvq3bt2mrRooX+8Y9/3PLPo2HDhlqzZo0WLlyob7/9VrGxsfL09FTLli01aNAgc7977rlH69at0+LFi/XVV1/p008/lZubm3x9fRUeHm61JF9UpTWui4uLPvroI82dO1dxcXHatGmTAgIC9OGHH+of//iH1ZnjKVOmKC0tTZ988okuXbokk8mkuLi4EgmNkhQaGqq33npLCxcu1Ntvv63atWsrPDxc9913n0aNGmVz/1GjRumbb75RTEyM0tPTVbt2bQUGBurpp5+2eFB9aRzHsGHDlJGRoY8//linTp1S/fr19fzzz2vEiBEW/dq1a6dXX31VixYt0uuvvy4fHx898cQT8vf3t7qZrH79+vrPf/6jJUuW6N///rdyc3MVHh5+w9Do7OyspUuX6t1339WXX36pbdu2ycPDQ6GhoZo8ebJFeAXskYOpLK6OBlDhrV27VjNnztSyZcvUvn378i6nUrt69ao6dOiggIAALV26tLzLAQBJXNMIAOWqsOtQP/30U126dMn8phsAqAhYngaAcvTCCy8oJydHwcHBcnFx0d69e7VhwwY1btxYjz/+eHmXBwBmhEYAKEedO3fW8uXL9cMPP5ivAR04cKAmTZpkfkc1AFQEXNMIAAAAm7imEQAAADYRGgEAAGAT1zSWkAsXMpWfz0p/RVS7trvOncso7zJwG5hD+8b82T/m0P4VzKGjo4O8vNxs71AIQmMJyc83ERorMObG/jGH9o35s3/Mof273TlkeRoAAAA2ERoBAABgE6ERAAAANhEaAQAAYBOhEQAAADYRGgEAAGAToREAAAA2ERoBAABgE6ERAAAANvFGGFRYAYGtlHL6pM1+des1UML+Q2VQEQAAlRehERVWyumT6jP1c5v9NswLK4NqAACo3FieBgAAgE2ERgAAANhEaAQAAIBNhEYAAADYRGgEAACATYRGAAAA2ERoBAAAgE2ERgAAANhEaAQAAIBNhEYAAADYRGgEAACATYRGAAAA2ERoBAAAgE2ERgAAANhUpbwLAG6Xo5OzfHw8bfarW6+BEvYfKoOKAAC48xAaUeYCAlsp5fTJEhsv/2qu+kz93Ga/DfPCSuwzAQCobAiNKHMpp08S8gAAsDPlek1jamqqIiMjFRERoeDgYBkMBsXHx1v169atmwwGg9VXZGSkVd9Lly5p1qxZ6tChg4KCgjR8+HAlJiYW+vlxcXEKDw9XmzZt1KVLF0VFRSkvL6/EjxMAAMDeleuZxqNHj2rJkiVq3LixDAaD9u7de8O+rVu31ogRIyza/Pz8LL7Pz8/X2LFjdeTIEY0aNUpeXl765JNPFBERobVr16pRo0bmvt98840mTJigDh06aNasWTpy5IgWLlyoCxcuaNasWSV7oAAAAHauXENj69attWvXLnl5eWn79u2aMGHCDfvWrVtX/fr1u+l4mzdv1t69e7Vw4UKFhIRIknr16qWePXsqKipKc+fONfedO3euWrVqpaVLl8rJyUmS5ObmpujoaEVERKhJkya3f4AAAAB3iHJdnnZ3d5eXl1eR++fk5OjKlSs33L5lyxb5+Pioe/fu5rZatWqpV69e2r59u3JzcyVJSUlJSkpK0qBBg8yBUZKGDBmi/Px8bd269RaOBgAA4M5lN89p3LFjh4KCghQUFKSQkBCtXLnSqk9iYqJat24tBwcHi/Y2bdooMzNTx48flyQdOnTtsSv+/v4W/erUqaO6deuatwMAAOAau7h72s/PT/fee6+aNGmiCxcuaNWqVfrnP/+ptLQ0jR071tzPaDSqQ4cOVvv7+PhIunbjTfPmzWU0GiVJ3t7eVn29vb2Vmppa7Bpr13Yv9j4oe97eHuVdAm4Rc2ffmD/7xxzav9udQ7sIjYsWLbL4vn///hoyZIjeeecdPfHEE/LwuPZDyMrKkouLi9X+BW1ZWVkW/yysr6ur602XwG/k3LkM5eebir0fypbRmF7eJeAWeHt7MHd2jPmzf8yh/SuYQ0dHh1s+0WU3y9PXc3Jy0ogRI3TlyhWLO66rVq2qnJwcq/4FbVWrVrX4Z2F9s7OzzdsBAABwjV2GRuna3dSSlJaWZm670dJyQVvBMnXBsnTBMvX1jEajuR8AAACusdvQeOLECUnX7o4u0LJlSx08eFAmk+UycUJCgqpXr25+TuM999wjSTpw4IBFvzNnziglJcW8HQAAANdU+NB48eJF5efnW7RlZ2dr6dKlcnNzU1BQkLk9NDRUqampiouLM7edP39emzdvVvfu3eXs7CxJatGihZo1a6aVK1fq6tWr5r4rVqyQo6OjevToUcpHBQAAYF/K/UaYd955R5KUnJwsSYqNjdXu3bvl6empYcOG6auvvtKiRYvUs2dP+fr66uLFi1q3bp2OHTumF198UW5ubuaxevbsqaCgIE2bNs38RpgVK1YoPz9fEydOtPjcadOmafz48Ro9erR69+6tI0eOaPny5Ro0aJCaNm1adj8AAAAAO1DuoXHBggUW369Zs0aS5Ovrq2HDhsnPz0/NmjVTbGyszp8/LxcXF7Vu3VozZsxQ165dLfZ1cnJSdHS05s6dq5iYGGVnZ6tNmzZ67bXX1LhxY4u+Xbt2VVRUlKKiojR79mzVqlVL48eP1zPPPFO6BwwAAGCHyj00Hj58+Kbb/f39rR65czM1atTQK6+8oldeecVm35CQEPPrBgEAAHBjFf6aRgAAAJQ/QiMAAABsIjQCAADAJkIjAAAAbCI0AgAAwCZCIwAAAGwiNAIAAMAmQiMAAABsIjQCAADAJkIjAAAAbCI0AgAAwCZCIwAAAGwiNAIAAMCmKuVdAFBWHJ2c5ePjabNf3XoNlLD/UBlUBACA/SA0otLIv5qrPlM/t9lvw7ywMqgGAAD7wvI0AAAAbCI0AgAAwCaWp1FiAgJbKeX0yfIuAwAAlAJCI0pMyumTXDMIAMAdiuVpAAAA2ERoBAAAgE2ERgAAANhEaAQAAIBNhEYAAADYRGgEAACATeX6yJ3U1FQtW7ZM+/fv14EDB3T58mUtW7ZM7du3N/e5cOGC1qxZo6+++kq///678vLy1Lx5c40cOVK9evWyGG/t2rWaOXNmoZ+VkJAgV1dXi7a4uDhFRUUpKSlJtWvX1mOPPaZx48apShWeRAQAAHC9ck1HR48e1ZIlS9S4cWMZDAbt3bvXqs++ffv05ptv6qGHHtL48eNVpUoVbdmyRZMnT9bvv/+uCRMmWO0zZcoU1atXz6LN2dnZ4vtvvvlGEyZMUIcOHTRr1iwdOXJECxcu1IULFzRr1qySPVAAAAA7V66hsXXr1tq1a5e8vLy0ffv2QgPg3XffrS1btsjX19fcNmTIEI0cOVLR0dEaPXq0qlatarHPww8/rHvuueemnz137ly1atVKS5culZOTkyTJzc1N0dHRioiIUJMmTW7/AAEAAO4Q5XpNo7u7u7y8vG7ap2HDhhaBUZIcHBwUEhKirKws/fnnn4Xul5GRofz8/EK3JSUlKSkpSYMGDTIHRulaGM3Pz9fWrVuLeSSojAICW8nHx7NIXwGBrcq7XAAAbovdXrx39uxZSSo0dA4ZMkSXL1+Wq6urunTpohkzZqh+/frm7YcOHZIk+fv7W+xXp04d1a1b17wduJmivjZR4tWJAAD7Z5eh8eLFi/rss890//33q1atWub2atWqqX///mrfvr3c3Ny0f/9+ffTRR9q/f7/WrVtn7ms0GiVJ3t7eVmN7e3srNTW1bA4EAADATthdaMzPz9f//d//KT09XS+88ILFtl69elncUf3II4/ovvvu09ixY/XRRx9pypQpkqSsrCxJkouLi9X4rq6uunLlSrHrql3bvdj7oOLy9vawizErE35+9o35s3/Mof273Tm0u9A4e/Zsff/994qMjJTBYLDZ/+GHH1azZs30ww8/mENjwY0zOTk5Vv2zs7OtbqwpinPnMpSfbyr2fqiYjMZ0uxizsvD29uDnZ8eYP/vHHNq/gjl0dHS45RNddvVw76ioKH3yySd67rnn1KdPnyLvV69ePaWlpZm/L1iWLlimvp7RaJSPj8/tFwsAAHAHsZvQuHz5cr399tsaOXKkRo8eXax9T5w4YXHDTMHjeA4cOGDR78yZM0pJSbH5uB4AAIDKxi5C45dffqmXX35Zffv21YwZM27Y7/z581Zt69ev1/Hjx9W5c2dzW4sWLdSsWTOtXLlSV69eNbevWLFCjo6O6tGjR8keAAAAgJ0r92sa33nnHUlScnKyJCk2Nla7d++Wp6enhg0bpoSEBE2bNk01a9ZUx44d9cUXX1js36lTJ911112SpMGDB6t169Zq1aqV3N3dlZCQoM8//1xNmjTRiBEjLPabNm2axo8fr9GjR6t37946cuSIli9frkGDBqlp06ZlcOQAAAD2o9xD44IFCyy+X7NmjSTJ19dXw4YNU1JSknJzc3X+/Hk9//zzVvsvW7bMHBp79eql//73v/ruu++UlZUlHx8fDR06VM8++6w8PCzvGOratauioqIUFRWl2bNnq1atWho/fryeeeaZUjpSAAAA+1XuofHw4cM33d6/f3/179+/SGNNmTLFfId0UYSEhCgkJKTI/QEAACoru7imEQAAAOWL0AgAAACbCI0AAACwidAIAAAAmwiNAAAAsInQCAAAAJsIjQAAALCJ0AgAAACbCI0AAACwidAIAAAAmwiNAAAAsInQCAAAAJsIjQAAALCJ0AgAAACbCI0AAACwidAIAAAAm6qUdwFAZeDo5CwfH0+b/erWa6CE/YfKoCIAAIqH0AiUgfyrueoz9XOb/TbMCyuDagAAKD6WpwEAAGAToREAAAA2ERoBAABgE6ERAAAANhEaAQAAYFOx7p6+cOGCzp8/r+bNm5vbTpw4oQ8//FAXL15UWFiYHnzwwRIvEgAAAOWrWKHxlVde0bFjx7R69WpJUmZmpoYOHarU1FRJ0qZNm/TRRx/pvvvuK/lKAQAAUG6KtTy9b98+Pfzww+bvv/zyS6Wmpio6Olrfffedmjdvrvfee6/I46WmpioyMlIREREKDg6WwWBQfHx8oX3j4uIUHh6uNm3aqEuXLoqKilJeXp5Vv0uXLmnWrFnq0KGDgoKCNHz4cCUmJt7WmAAAAJVdsULjuXPnVLduXfP33333nfz9/fXQQw/J29tb4eHhOnSo6G+zOHr0qJYsWaIzZ87IYDDcsN8333yjCRMmqEaNGpo1a5ZCQkK0cOFCzZkzx6Jffn6+xo4dq40bN2rYsGF67rnndO7cOUVEROj48eO3NCYAAACKuTxdpUoVZWdnm7//8ccfFR4ebv7ew8NDFy9eLPJ4rVu31q5du+Tl5aXt27drwoQJhfabO3euWrVqpaVLl8rJyUmS5ObmpujoaEVERKhJkyaSpM2bN2vv3r1auHChQkJCJEm9evVSz549FRUVpblz5xZ7TAAAABTzTGOTJk20ZcsWmUwmxcXFKS0tTR07djRvT0lJUY0aNYo8nru7u7y8vG7aJykpSUlJSRo0aJA53EnSkCFDlJ+fr61bt5rbtmzZIh8fH3Xv3t3cVqtWLfXq1Uvbt29Xbm5usccEAABAMUPj0KFD9dNPP+m+++7TpEmT1LBhQ4vQ+PPPP990mflWFCx3+/v7W7TXqVNHdevWtVgOT0xMVOvWreXg4GDRt02bNsrMzDQvURdnTAAAABRzeTosLEzStRtI3N3dNW7cODk7O0u69jie9PR0PfHEEyVaoNFolCR5e3tbbfP29jbfuV3Qt0OHDlb9fHx8JF278aZ58+bFGhMAAGYPJ+sAACAASURBVADFDI3SteBYEB6v5+XlpbVr15ZIUdfLysqSJLm4uFhtc3V11ZUrVyz6FtavoK1grOKMWVS1a7sXex9UXN7eHpXysysyfi72jfmzf8yh/bvdOSxWaOzevbuef/55i2sGr/f111/r5ZdfVlxc3G0Vdb2qVatKknJycqy2ZWdnm7cX9C2sX0FbQd/ijFlU585lKD/fVOz9UDEZjemV8rMrKm9vD34udoz5s3/Mof0rmENHR4dbPtFVrGsa//zzT12+fPmG269cuaJTp07dUiE3UrCEXLCkfD2j0Wheei7oW9jSckFbQd/ijAkAAIASfvf02bNnb+ks3c3cc889kqQDBw5YtJ85c0YpKSnm7ZLUsmVLHTx4UCaT5Rm/hIQEVa9eXY0aNSr2mAAAACjC8vRPP/1k8ZaWbdu26Y8//rDql5aWpi+//LLEA1eLFi3UrFkzrVy5Uo899pj5ETkrVqyQo6OjevToYe4bGhqqLVu2KC4uzvycxvPnz2vz5s3q3r27+aad4owJAACAIoTG+Ph4RUVFSZIcHBy0devWGz7HsHHjxpo5c2axCnjnnXckScnJyZKk2NhY7d69W56enho2bJgkadq0aRo/frxGjx6t3r1768iRI1q+fLkGDRqkpk2bmsfq2bOngoKCNG3aNI0aNUpeXl5asWKF8vPzNXHiRIvPLeqYAAAAKEJoHDFihMLDw2UymRQSElLojTAODg6qXr26atasWewCFixYYPH9mjVrJEm+vr7m0Ni1a1dFRUUpKipKs2fPVq1atTR+/Hg988wzFvs6OTkpOjpac+fOVUxMjLKzs9WmTRu99tpraty4sUXfoo4JAACAIoRGDw8PeXhcu0V72bJlat68uWrXrl1iBRw+fLhI/UJCQsxLzjdTo0YNvfLKK3rllVdKbEwAAIDKrliP3Ln//vtLqw4AAABUYMV+uPepU6e0cuVKHTt2TBcvXrS6U9nBwUEfffRRiRUIAACA8les0PjNN9/o2WefVW5u7i1fwwgAAAD7U6zQOG/ePHl5eWnhwoVq06ZNadUEAACACqZYD/f+/fffNWLECAIjAABAJVOs0FirVi3zA7IBAABQeRQrNPbr1++GD/YGAADAnatY1zSGh4crPj5e48eP1/Dhw9WgQQPzK/iuV79+/RIrEAAAAOWvWKGxV69ecnBwkMlk0n//+98b9ktMTLzdugAAAFCBFCs0TpgwQQ4ODqVVC1DpOTo5y8fH02a/uvUaKGH/oTKoCACAa4oVGidOnFhadaCCCghspZTTJ8u7jEoj/2qu+kz93Ga/DfPCyqAaAAD+p9hvhEHlknL6ZJFCjESQAQDgTlas0PjTTz8Vqd999913S8UAAACgYipWaIyIiCjSNY3cCAMAAHBnKVZonDNnjlVbXl6eTpw4obVr16pBgwYaNGhQiRUHAACAiqHYz2m8kdGjR990OwAAAOxXsd4IczM1atTQwIED9d5775XUkAAAAKggSiw0SpKnp6dOnDhRkkMCAACgAiix0Jidna0vvvhCd911V0kNCQAAgAqiWNc0zpw5s9D2tLQ07du3T+fPn9e0adNKpDAAAABUHMUKjevWrSu0vUaNGmratKlmzpypvn37lkhhAAAAqDiKFRp//fXX0qoDAAAAFViJ3ggDAACAO9MtvXs6IyNDO3fuNN8p3bBhQz3wwANyd3cv0eIAAABQMRQ7NH722Wd69dVXdfnyZZlMJkmSg4ODqlevrhkzZmjgwIElXiQAAADKV7FCY1xcnGbNmqWGDRtq0qRJatGihSTpt99+08cff6x//vOfql27trp161YqxQIAAKB8FCs0vvfee2revLlWrVolNzc3c3vHjh3Vv39/DRo0SEuWLCnx0Dhjxowb3rktSd9++63q1KmjiIgI/fjjj1bbe/furfnz51u05eTkaMGCBYqNjdWlS5fUsmVLTZkyRR07dizR2gEAAO4Exb57esKECRaBsYC7u7vCwsL0zjvvlFhxBQYNGmQV5kwmk1588UX5+vqqTp065vb69etr8uTJFn19fX2txpwxY4a2bt2q4cOHq3Hjxlq3bp3GjBmjmJgYBQcHl/gxAAAA2LNbuhHmRhwcHEpyOLPg4GCrIPfzzz/rypUrVs+F9PT0VL9+/W46XkJCgjZu3KiZM2dq5MiRkqSwsDD16dNHkZGRWr58eYnWDwAAYO+K9cgdg8GgdevW6fLly1bbMjMztW7dOrVs2bLEiruZDRs2yMHBQX369LHalpeXp8zMzBvuu3nzZjk7O1vctOPq6qrHHntMu3fvVmpqaqnUDAAAYK+Kdabxqaee0rPPPqvw8HANHz5czZs3lyQlJSUpJiZGx48f19tvv10qhV4vNzdXmzZtUnBwsBo0aGCxLTk5WUFBQcrNzZW3t7eGDRumsWPHytHxf/k4MTFRTZs2tVpmDwgIkMlkUmJionx8fEr9OAAAAOxFsUJjSEiIZs2apcjISM2ePdu8HG0ymVStWjXNmjVLISEhpVLo9b7//ntdvHjRamm6YcOGat++vQwGgzIyMrRhwwbNnz9fp06d0ksvvWTuZzQaLa6DLODt7S1Jt3SmsXZtnlF5J/H29ijvEmyyhxpLUmU73jsN82f/mEP7d7tzWOxrGocOHaq+fftqx44dOnnypKRrYa1Tp07y8CibP1AbNmyQs7OzevXqZdH+n//8x+L78PBwTZo0SatWrdLIkSPVrFkzSVJWVpacnZ2txnV1dZUkZWdnF7umc+cylJ9vKvZ+qJiMxvTyLsEme6ixpHh7e1Sq473TMH/2jzm0fwVz6OjocMsnum7pRhhPT0+rwFZWMjMzFRcXp86dO8vLy8tm/1GjRmnz5s2Kj483h8aqVasqNzfXqm9BWCwIjwAAALjGZmi8evWq5s+fL19fXz3xxBM37PfJJ58oJSVFU6ZMKbW7qCVp+/bthd41fSN169aVJKWlpZnbvL29C12CNhqNksT1jKjwHJ2c5ePjabNf3XoNlLD/UBlUBAC409kMjV988YWWLl2qzz777Kb9AgICNHv2bLVo0aLIge5WrF+/XtWrVy/yA8QL3o9dq1Ytc1vLli0VExOjzMxMi5th9u/fb94OVGT5V3PVZ+rnNvttmBdWBtUAACoDm4/c2bRpkx544AH5+/vftJ+/v786d+6sjRs3llhxf3X+/Hn98MMPeuSRR1StWjWLbRkZGcrJybFou3r1qhYvXixHR0eLh4OHhoYqNzfXIgjn5ORo7dq1atu2baE3yQAAAFRmNs80Hjx4UE8++WSRBmvfvr0+/PDD263phr788kvl5eUVeibz4MGD+vvf/64+ffqoUaNGunz5sjZt2qQDBw5ozJgxatiwoblvYGCgQkNDFRkZKaPRqEaNGmndunU6deqU5syZU2r1AwAA2CuboTEtLU21a9cu0mC1atXSxYsXb7uoG1m/fr1q166tBx54wGpb/fr11bZtW23dulVnz56Vo6OjWrRooVdffVXh4eFW/efOnas333xTsbGxSktLk8FgUHR0tNq1a1dq9QMAANgrm6HRzc1NFy5cKNJgFy9eLPS91CVl5cqVN9zWsGFDvfXWW0Uey9XVVdOnT9f06dNLojQAAIA7ms1rGu+++27t2LGjSIPt2LFDd999920XBQAAgIrFZmh85JFHtHPnTm3fvv2m/eLi4rRz50716NGjxIoDAABAxWAzNA4ePFiNGjXS5MmTNX/+fPNbYAqcPHlS8+fP1+TJk9WkSRMNHjy41IoFAABA+bB5TWPVqlUVHR2tp59+WosXL1Z0dLTc3d3l5uamzMxMZWRkyGQyqWnTplq8eDFvUwEAALgDFek1go0bN1ZsbKxWrVqlLVu26LffftPZs2fl5uame++9Vz169NDAgQNVtWrV0q4XAAAA5aDI7552dXVVRESEIiIiSrMeAAAAVEA2r2kEAAAACI0AAACwidAIAAAAmwiNAAAAsInQCAAAAJsIjQAAALCJ0AgAAACbCI0AAACwqcgP9wZgfxydnOXj42mzX916DZSw/1AZVAQAsFeERuAOln81V32mfm6z34Z5YWVQDQDAnrE8DQAAAJsIjQAAALCJ0AgAAACbCI0AAACwidAIAAAAmwiNAAAAsInQCAAAAJsIjQAAALCJ0AgAAACb7CI0xsfHy2AwFPqVnJxs0XfPnj164oknFBgYqE6dOunll1/WlStXrMbMycnR66+/rs6dOysgIECPP/64fvjhh7I6JAAAALtiV68RHDFihFq3bm3RVqdOHfO/JyYmauTIkbr77rs1Y8YMpaSk6P3339fJkye1aNEii/1mzJihrVu3avjw4WrcuLHWrVunMWPGKCYmRsHBwWVyPOUpILCVUk6fLO8yAACAnbCr0Hj//fcrJCTkhtvnzZunmjVrKiYmRm5ubpKkBg0a6IUXXtAPP/ygjh07SpISEhK0ceNGzZw5UyNHjpQkhYWFqU+fPoqMjNTy5ctL/VjKW8rpk7yTGAAAFJldLE9fLyMjQ3l5eYW279y5U2FhYebAKEn9+vVT9erVtWnTJnPb5s2b5ezsrIEDB5rbXF1d9dhjj2n37t1KTU0t3YMA7FRAYCv5+Hja/AoIbFXepQIASphdnWl87rnndPnyZVWpUkXt27fX9OnTZTAYJEmHDx9WXl6e/P39LfZxcXHRPffco8TERHNbYmKimjZtahEuJSkgIEAmk0mJiYny8fEp/QMC7AxnqAGg8rKL0Ojs7KyePXvqoYcekpeXlw4fPqz3339fQ4YM0erVq9W0aVMZjUZJkre3t9X+3t7e2rdvn/l7o9FocS3k9f0k3dKZxtq13Yu9Dyoub2+P8i6hzJX0MVf08VC2mD/7xxzav9udQ7sIjW3btlXbtm3N33fv3l3dunXTgAEDFBUVpTfeeENZWVmSrp1Z/CtXV1fzdknKysqSs7Nzof0kKTs7u9g1njuXofx8U7H3Q8VkNKaXdwllrqSPuSTH8/b2qJRzcqdg/uwfc2j/CubQ0dHhlk902d01jQVatmypjh07ateuXZKkqlWrSrr2KJ2/ys7ONm8v6Jubm1toP+l/4REAAADX2MWZxhupV6+eOTQWLC0XLFNfz2g0Wlyj6O3tXegSdMG+XM+IysbRyVk+Pp7lXQYAoAKz69B44sQJeXl5SZL8/PxUpUoVHThwQD169DD3ycnJUWJiovr27Wtua9mypWJiYpSZmWlxM8z+/fvN24HKJP9qLje4AABuyi6Wp8+fP2/V9vPPPys+Pl6dO3eWJHl4eKhjx46KjY1VZmamuV9sbKwuX76s0NBQc1toaKhyc3P12WefmdtycnK0du1atW3bttCbZAAAACozuzjTOHnyZFWrVk3BwcHy8vLSb7/9ppUrV8rLy0sTJ04095syZYoGDx6siIgIDRw4UCkpKfrggw/00EMP6YEHHjD3CwwMVGhoqCIjI2U0GtWoUSOtW7dOp06d0pw5c8rjEAEAACo0uwiNISEhWr9+vT744ANlZGSoVq1a6tOnjyZOnKj69eub+7Vu3VoffPCBIiMjNWfOHLm7u+vxxx/X1KlTrcacO3eu3nzzTcXGxiotLU0Gg0HR0dFq165dWR4aAACAXbCL0Dh8+HANHz68SH3vvfdeffrppzb7ubq6avr06Zo+ffrtlgcAAHDHs4trGgEAAFC+CI0AAACwidAIAAAAmwiNAAAAsMkuboRB0QUEtlLK6ZPlXQYAALjDEBrvMCmnT/JmDwAAUOJYngYAAIBNhEYAAADYRGgEAACATYRGAAAA2ERoBAAAgE2ERgAAANhEaAQAAIBNPKcRQIlzdHKWj49nkfrWrddACfsPlXJFAIDbRWgEUOLyr+YW6SHzEg+aBwB7wfI0AAAAbCI0AgAAwCZCIwAAAGwiNAIAAMAmQiMAAABsIjQCAADAJkIjAAAAbCI0AgAAwCZCIwAAAGzijTAAylVRXznI6wYBoHzZRWhMSEjQunXrFB8fr1OnTqlmzZoKDg7W5MmT1bhxY3O/iIgI/fjjj1b79+7dW/Pnz7doy8nJ0YIFCxQbG6tLly6pZcuWmjJlijp27FjqxwPgf4r6ykFeNwgA5csuQuN7772nPXv2KDQ0VAaDQUajUcuXL1dYWJhWr16t5s2bm/vWr19fkydPttjf19fXaswZM2Zo69atGj58uBo3bqx169ZpzJgxiomJUXBwcKkfEwAAgD2xi9A4cuRIRUZGysXFxdzWu3dv9e3bV0uWLNGrr75qbvf09FS/fv1uOl5CQoI2btyomTNnauTIkZKksLAw9enTR5GRkVq+fHmpHAcAAIC9sosbYdq2bWsRGCWpSZMmatGihZKTk6365+XlKTMz84bjbd68Wc7Ozho4cKC5zdXVVY899ph2796t1NTUkiseAADgDmAXobEwJpNJZ8+elZeXl0V7cnKygoKC1LZtW3Xu3FmLFi1Sfn6+RZ/ExEQ1bdpUbm5uFu0BAQEymUxKTEws9foBAADsiV0sTxfmiy++0JkzZzRlyhRzW8OGDdW+fXsZDAZlZGRow4YNmj9/vk6dOqWXXnrJ3M9oNKpOnTpWY3p7e0vSLZ1prF3b/RaOAhWVt7dHeZeAQjAv9ou5s3/Mof273Tm0y9CYnJysl156Se3atbO4fvE///mPRb/w8HBNmjRJq1at0siRI9WsWTNJUlZWlpydna3GdXV1lSRlZ2cXu6Zz5zKUn28q9n6omIzG9PIuAYVgXuyTt7cHc2fnmEP7VzCHjo4Ot3yiy+6Wp41Go55++mnVqFFDCxYskKPjzQ9h1KhRMplMio+PN7dVrVpVubm5Vn0LwmJBeAQAAMA1dnWmMT09XWPGjFF6erpWrFhhXk6+mbp160qS0tLSzG3e3t6FLkEbjUZJko+PTwlVDAAAcGewmzON2dnZGjdunI4dO6bFixebl5ptOXHihCSpVq1a5raWLVvq6NGjVndY79+/37wdAAAA/2MXofHq1auaPHmy9u3bpwULFigoKMiqT0ZGhnJycqz2W7x4sRwdHS3e9BIaGqrc3Fx99tln5racnBytXbtWbdu2LfQmGQAAgMrMLpanX331VX311Vfq2rWrLl68qNjYWPM2Nzc3hYSE6ODBg/r73/+uPn36qFGjRrp8+bI2bdqkAwcOaMyYMWrYsKF5n8DAQIWGhioyMlJGo1GNGjXSunXrdOrUKc2ZM6c8DhFACQkIbKWU0ydt9uNd1gBQPHYRGn/99VdJ0tdff62vv/7aYpuvr69CQkJUv359tW3bVlu3btXZs2fl6OioFi1a6NVXX1V4eLjVmHPnztWbb76p2NhYpaWlyWAwKDo6Wu3atSuTYwJQOlJOn+Rd1gBQCuwiNMbExNjs07BhQ7311ltFHtPV1VXTp0/X9OnTb6c0AACASsEuQiMAlDRHJ2f5+Hja7McyNgBcQ2gEUCnlX80t0jL2lwsGEi4BQIRGALipooZLrpEEcKezi0fuAAAAoHwRGgEAAGAToREAAAA2ERoBAABgE6ERAAAANnH3NACUAJ77COBOR2gEgBLAo3kA3OlYngYAAIBNhEYAAADYxPI0AJShol77KHH9I4CKhdAIAGWoqNc+Slz/CKBiYXkaAAAANhEaAQAAYBOhEQAAADZxTSMAVFA8MBxARUJoBIAKigeGA6hIWJ4GAACATZxpBAA7V9Rl7CrOVZWXm2WzH8vdAApDaAQAO1ecZWyWuwHcKkIjAKBU+TZopFN/nrDZjzOcQMVGaAQAWCjpu7ZP/XmiSGc4v1wwkFcsAhUYoREAYKG87trmFYtAxVap757OycnR66+/rs6dOysgIECPP/64fvjhh/IuCwAAoMKp1GcaZ8yYoa1bt2r48OFq3Lix1q1bpzFjxigmJkbBwcHlXR4A4Aa4Yxwoe5U2NCYkJGjjxo2aOXOmRo4cKUkKCwtTnz59FBkZqeXLl5dvgQCAGyrpO8aLej0l4RKVWaUNjZs3b5azs7MGDhxobnN1ddVjjz2m+fPnKzU1VT4+PuVYIQCgrBQ1hBY1XBb1DCdnQmFPKm1oTExMVNOmTeXm5mbRHhAQIJPJpMTExGKFRkdHh5Iu8ZY0btxYPl7VyrxfeX52afQrynxW1p9Nef65Kcl5qej97KHG8pqT0hizqP0aNqiv7k9F2+wX997YEu339QcTdN99bWz2kyRvn3ravGlbkfoWRWivR2RMPW2zXxVnV+XlZtvsV9T6ivq5JX28paGiHIujo8Nt5RUHk8lkKsF67EafPn1Up04dLV261KI9KSlJf/vb3/Tyyy9bnIUEAACozCrt3dNZWVlydna2and1dZUkZWfb/m0JAACgsqi0obFq1arKzc21ai8IiwXhEQAAAJU4NHp7eys1NdWq3Wg0ShI3wQAAAFyn0obGli1b6ujRo8rMzLRo379/v3k7AAAArqm0oTE0NFS5ubn67LPPzG05OTlau3at2rZtqzp16pRjdQAAABVLpX3kTmBgoEJDQxUZGSmj0ahGjRpp3bp1OnXqlObMmVPe5QEAAFQolfaRO9K1m17efPNNrV+/XmlpaTIYDJo6daoeeOCB8i4NAACgQqnUoREAAABFU2mvaQQAAEDRERoBAABgE6ERdi01NVWRkZGKiIhQcHCwDAaD4uPjC+0bFxen8PBwtWnTRl26dFFUVJTy8vLKuGJcLyEhQf/+97/Vu3dvBQUFqUuXLpoyZYr++OMPq7579uzRE088ocDAQHXq1Ekvv/yyrly5Ug5V43q//PKLJkyYoK5duyogIECdOnXS6NGjtWfPHqu+zKF9WLJkiQwGg/r162e1jTmsmOLj42UwGAr9Sk5Otuh7O3NYae+exp3h6NGjWrJkiRo3biyDwaC9e/cW2u+bb77RhAkT1KFDB82aNUtHjhzRwoULdeHCBc2aNauMq0aB9957T3v27FFoaKgMBoOMRqOWL1+usLAwrV69Ws2bN5ckJSYmauTIkbr77rs1Y8YMpaSk6P3339fJkye1aNGicj6Kyu3EiRO6evWqBg4cKG9vb6Wnp2v9+vUaNmyYlixZok6dOkliDu2F0WjUu+++q+rVq1ttYw4rvhEjRqh169YWbdc/QvC259AE2LH09HTT+fPnTSaTybRt2zaTn5+fadeuXVb9evfubQoPDzfl5eWZ2+bNm2dq2bKl6ejRo2VVLv5i9+7dpuzsbIu2o0ePmvz9/U3Tp083tz311FOmBx980JSRkWFuW7VqlcnPz8+0c+fOMqsXRXP58mXTAw88YBo7dqy5jTm0D9OnTzdFRESYhg0bZnr00UcttjGHFdeuXbtMfn5+pm3btt203+3OIcvTsGvu7u7y8vK6aZ+kpCQlJSVp0KBBcnJyMrcPGTJE+fn52rp1a2mXiRto27atXFxcLNqaNGmiFi1amJdUMjIytHPnToWFhcnNzc3cr1+/fqpevbo2bdpUpjXDtmrVqqlWrVq6dOmSJObQXiQkJOiLL77QzJkzrbYxh/YjIyOj0EuvSmIOCY244x06dEiS5O/vb9Fep04d1a1b17wdFYPJZNLZs2fNvwwcPnxYeXl5VvPn4uKie+65R4mJieVRJv4iIyND58+f1++//6558+bpyJEj6tixoyTm0B6YTCbNnj1bYWFhuueee6y2M4f24bnnnlO7du0UGBioUaNG6fDhw+ZtJTGHXNOIO57RaJQkeXt7W23z9vZWampqWZeEm/jiiy905swZTZkyRZLt+du3b1+Z1ofCPf/889qyZYskydnZWYMHD9a4ceMkMYf24PPPP1dSUpIWLlxY6HbmsGJzdnZWz5499dBDD8nLy0uHDx/W+++/ryFDhmj16tVq2rRpicwhoRF3vKysLEmyWgaVJFdXV+78q0CSk5P10ksvqV27duY7N23NX8F2lK8JEyZo0KBBSklJUWxsrHJycpSbmysXFxfmsILLyMjQG2+8obFjx8rHx6fQPsxhxda2bVu1bdvW/H337t3VrVs3DRgwQFFRUXrjjTdKZA5ZnsYdr2rVqpKknJwcq23Z2dnm7ShfRqNRTz/9tGrUqKEFCxbI0fHa/56YP/tgMBjUqVMnDRgwQEuXLtXBgwfN18YxhxXbu+++K2dnZz355JM37MMc2p+WLVuqY8eO2rVrl6SSmUNCI+54BafiC07NX89oNN7wN2uUnfT0dI0ZM0bp6el67733LJZPmD/74+zsrO7du2vr1q3KyspiDiuw1NRUffTRRxoyZIjOnj2rkydP6uTJk8rOzlZubq5OnjyptLQ05tBO1atXT2lpaZJK5v+lhEbc8Qou6j5w4IBF+5kzZ5SSklLoRd8oO9nZ2Ro3bpyOHTumxYsXq1mzZhbb/fz8VKVKFav5y8nJUWJiIvNXQWVlZclkMikzM5M5rMDOnTun3NxcRUZGqnv37uav/fv3Kzk5Wd27d9eSJUuYQzt14sQJ802FJTGHhEbc8Vq0aKFmzZpp5cqVunr1qrl9xYoVcnR0VI8ePcqxusrt6tWrmjx5svbt26cFCxYoKCjIqo+Hh4c6duyo2NhYZWZmmttjY2N1+fJlhYaGlmXJ+Ivz589btWVkZGjLli2qV6+eateuzRxWYA0aNNDChQutvlq0aCFfX18tXLhQYWFhzGEFV9h/hz///LPi4+PVuXNnSSXz/1IHk8lkKrmygbL3zjvvSLp2E8WGDRs0YMAANWjQQJ6enho2bJgk6euvv9b48ePVoUMH9e7dW0eOHNHy5cs1aNAgvfjii+VYfeX2yiuvaNmyZeratat69eplsc3NzU0hISGSpIMHD2rw4MFq0aKFBg4cqJSUFH3wwQdq3769lixZUh6l4/83fPhwubq6Kjg4WN7e3jp9+rTWrl2rlJQUzZs3T71795bEHNqbiIgIXbp0SbGxseY25rDiGj58uKpVq6bg4GB5eXnpt99+08qVK+Xh4aHVq1erfv36km5/wrXEZAAADLJJREFUDgmNsHsGg6HQdl9fX3311Vfm77dv366oqCglJyerVq1aGjBggJ555hlVqcJDBMpLRESEfvzxx0K3/XX+fv75Z0VGRurQoUNyd3dX7969NXXq1EJfd4ays3r1asXGxiopKUmXLl2Sh4eHgoKCNGrUKN1///0WfZlD+1FYaJSYw4pq2bJlWr9+vY4fP66MjAzVqlVLnTt31sSJE82BscDtzCGhEQAAADZxTSMAAABsIjQCAADAJkIjAAAAbCI0AgAAwCZCIwAAAGwiNAIAAMAmQiMAAABsIjQCACq8tWvXymAwKD4+vrxLASotXoUBoEycOHFC0dHR+umnn3T69Gm5uLjorrvuUkBAgMLDw9WhQ4fyLvGOFBERoQMHDmjv3r3lXYpNiYmJ2r59u8LDw9WgQYPyLgfAXxAaAZS6X375RREREapSpYrCwsJ09913KysrS3/88Yd27NghNzc3QiOUmJioqKgo3X///YRGoAIiNAIodQsXLtSVK1f+v/buP6bq6g3g+PsikULKVSNNFLD0XoJL9zJtZGBLVFDkNzhQLIvFRo3GwhyYZTpbRA1dUU5yoZNQsICM6ULDFiRz/UBsoNlCiR+LIAxDQAE53z8c9+vtXsMw0j+e18bG55znnM85n8vYwznnXjh48CCenp5W9R0dHbdhVEIIIf4JOdMohBhzjY2NaLVamwkjgIuLi1VZdXU1iYmJzJ8/Hx8fH8LCwti/f7/N9gcOHGDZsmUYDAaWLl3Knj17KC4utjoDl5GRgV6vt9mHXq8nIyPDqvzw4cOsWrUKX19fjEYjK1eu5PPPP79h+5MnT7JmzRpMJhN+fn5s3LiRnp4eq/iOjg5ef/11Fi9ejMFgYMGCBTzzzDMcP37cIq6xsZH169cTEBCAwWAgMDCQrKwsent7bc5jtJRS7Nu3j+joaIxGI76+vjz55JOcOHHCIq6lpQW9Xk9OTg5ffvklMTEx+Pj4EBAQQFZWFoODg1Z9l5eXEx4ejo+PD0888QTvvfce1dXV6PV6SkpKAMjJyWHDhg0APPXUU+j1epuvydDQEB9++CFLlizBYDAQHBxMaWnpv/oshBC2yUqjEGLMubm5cf78eY4cOUJQUNCI8UVFRbz22muYTCaSk5OZMGEC1dXVbN68maamJtLT082xe/bsITMzE09PT9LS0ujr6yMvL4+pU6fe8ri3b9/Ozp07WbhwIampqdjZ2XH06FFSU1PZtGkTCQkJFvFnzpwhOTmZ6OhoQkND+eabb/jkk0+ws7Nj69at5riWlhZWrVpFZ2cnERERGAwG+vr6OHXqFNXV1fj7+wNQV1fH2rVrmTRpEnFxcUybNo0ff/yR/Px8Tp48SX5+PnfdddctzxNg/fr1HDp0iODgYKKjo+nv76esrIzExERycnJYvHixRfxXX33Fvn37iI+PJyYmhoqKCvLy8nB2diY5Odkcd/jwYdLS0nBzcyMlJYVx48bx6aefcuzYMYv+li5dSkdHB0VFRSQnJ/PAAw8A1352rrd9+3YuX75MXFwcDg4O7N+/n4yMDNzc3Jg3b96/8iyEEDeghBBijNXU1Chvb2+l0+lUUFCQysjIUAUFBernn3+2iv3tt9+UwWBQaWlpVnVbt25Vnp6eqqmpSSml1MWLF5XRaFTLly9Xvb295rhff/1VmUwmpdPp1IkTJ8zl6enpSqfT2RyjTqdT6enp5uu6ujql0+lUdna2Vexzzz2nfH19VXd3t0V7vV6vamtrLWKTkpKUl5eXunTpkrns2WefVTqdTlVWVlr1ffXqVfP3YWFhKjg42OI+Sil15MgRpdPpVHFxsc25XG/NmjXKZDL9bcxwf4WFhRblAwMDKioqSi1atEgNDQ0ppZRqbm5WOp1OGY1G1dzcbI4dGhpSK1asUP7+/hbtAwIC1IIFC1RXV5e5/NKlSyowMNBqDsXFxVav2V/rIiIi1JUrV8zlbW1tytvbW7344osjPgshxK2R7WkhxJjz9fWluLiYqKgouru7KSkpYcuWLYSEhJCQkEBzc7M5try8nP7+fmJjY7lw4YLFV2BgIENDQ1RXVwPw9ddf09fXR0JCAhMmTDD3MX36dMLCwm5pzGVlZWg0GiIjI22Oo6enh9raWos2JpMJo9FoUfboo48yODhIa2srAF1dXVRVVbFw4UIWLlxodV87u2u/ls+ePcvZs2cJDQ2lv7/f4v7z5s3D0dHRait7tD777DOcnJxYsmSJxX3+/PNPAgMDaW1tpbGx0aLN4sWLLd6sotFo8PPzo6Ojw7wdX19fT3t7O1FRUTg7O5tjnZyciI+PH9VYV69ejYODg/l62rRpzJ4922p8Qoh/n2xPCyH+E3q9njfffBOA1tZWvv32Wz7++GO+++47nn/+eYqLi3FwcKChoQGAp59++oZ9/f7778C1bV7AvJV5vQcffPCWxtvQ0IBSiuXLl484jmGzZs2yitFqtcC1ZBGgqakJpRReXl4j3h+unfXLycm5qfuPVkNDAz09PTz22GM3jOns7GT27Nnm65Hm6uTkZH59rm83zFbZzbjRfYeTciHE2JGkUQjxn3N1dcXV1ZWIiAhWr15NTU0NP/zwA/Pnz0cpBUBWVhb33Xefzfa2EoebodFobJbbevOGUgqNRsOuXbsYN26czXZz5syxuL5R3HB/o5GYmGhzRRJg0qRJo+rzr5RSTJkyhezs7BvGzJ071+J6LOZ6M4ZXYoUQ/z1JGoUQt41Go8FoNFJTU0N7ezsAHh4eAEyePPlvV74A8/bouXPnWLBggUXd8Erd9Ya3SLu6usyrYoDF9vgwDw8PqqqqmDFjxi2vWl7Pzc0NjUbDmTNn/jbO3d0duJYkjfQcbpW7uzuNjY0YjUacnJz+tX5dXV0BOH/+vFWdrbIbJfVCiDuD/MkmhBhzx48ft7mad/nyZfO5vOHEbPny5Tg4OJCTk8Ply5et2nR3d9Pf3w+Av78/48ePp6CggL6+PnNMW1sbZWVlVm2HE9LhM5HDdu/ebRUbHh4OwLZt27h69apV/Wi3hrVaLY8//jiVlZVW44D/r9J5eXmh0+koLCy0mdQODg6at7xvVWRkJENDQ2zbts1m/WjnajAYcHFxobS0lIsXL5rLe3p6KCwstIp3dHQEsIgVQtw5ZKVRCDHmMjMz6erqIjAwEJ1Ox/jx482JXWNjI5GRkebPT5w+fTqbN2/mlVdeISQkhPDwcFxdXblw4QI//fQTX3zxBYcOHWLmzJk4OzuTmppKVlYW8fHxREZG0tfXR2FhIR4eHpw+fdpiHKGhoWzfvp1NmzZx7tw5tFotVVVV/PHHH1Zjfvjhh3nhhRfIyckhMjKS4OBgpk2bRnt7O/X19VRWVlJXVzeq5/Hqq69y+vRpkpKSiIyMxNvbmytXrnDq1ClcXV1Zv349Go2Gt956i7Vr1xIeHk5MTIzFf9I5evQoaWlpREdHj3i/gYEBduzYYbMuKCiIZcuWER0dzUcffUR9fT2LFi1i8uTJtLW1UVtbyy+//EJFRcU/nqe9vT3p6em89NJLrFy5ktjYWMaNG0dpaSlarZaWlhaL1UUfHx/s7OzYuXMnFy9exNHRkZkzZ1q9uUgIcXtI0iiEGHMZGRlUVFTw/fffU15eTnd3NxMnTkSn05GUlGSV+MTExODh4UFeXh5FRUV0d3ej1WqZPXs2qampFh8GnpiYiKOjI7t37yY7O5v777+fxMREJk6cyMsvv2zR7z333MMHH3xAZmYmubm5ODo6EhQUxNtvv80jjzxiNe6UlBQMBgP5+fns3buX3t5epk6dyty5c9m4ceOon8esWbMoLi7m/fffp7KykoMHDzJp0iQ8PT2Ji4szxz300EOUlpaSm5vLsWPHKCwsxMnJCVdXV6Kioqy25G9kYGCAd955x2adu7s7c+bMITMzEz8/Pw4cOEBubi4DAwO4uLjg5eXFunXrRj3XsLAw7O3t2bFjB++++y733nsvsbGx6PV6UlJSuPvuu82xM2bM4I033mDXrl1s2bKFgYEBoqKiJGkU4g6hUWN5YlkIIW6TkpISNmzYwN69e/Hz87vdwxF/kZeXR1ZWFkVFRZhMpts9HCHETZAzjUIIIcZMf3+/1ZnQnp4eCgoK0Gq1I370kBDiziHb00IIIcZMc3MzSUlJrFixgpkzZ9LR0UFpaSktLS1s3rzZ4oO6hRB3NkkahRBCjJkpU6ZgMpkoKyujs7MTe3t7dDod69atIyQk5HYPTwjxD8iZRiGEEEIIMSI50yiEEEIIIUYkSaMQQgghhBiRJI1CCCGEEGJEkjQKIYQQQogRSdIohBBCCCFGJEmjEEIIIYQY0f8At7K80H5mbwcAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X0g_DhmbfAkn",
        "colab_type": "text"
      },
      "source": [
        "## Formatting data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DrKhrkaScV9d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_dataloader      = prepare_data(X_train, y_train, True)\n",
        "dev_dataloader        = prepare_data(X_dev, y_dev, False)\n",
        "prediction_dataloader = prepare_data(X_test, y_test, False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XwshuBdPSrHS",
        "colab_type": "text"
      },
      "source": [
        "## Main"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gzDwGM2lcqN9",
        "colab_type": "code",
        "outputId": "fe54c56e-78cf-4446-dade-61044356ba6e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 710
        }
      },
      "source": [
        "adam         = AdamW(model.parameters(), lr=LEARN_RATE, eps=EPSILON)\n",
        "total_steps  = len(train_dataloader) * EPOCHS # Total number of training steps is nb_batches times nb_epochs.\n",
        "linear_sch   = get_linear_schedule_with_warmup(adam, num_warmup_steps=nb_warmup_steps, num_training_steps=total_steps)\n",
        "\n",
        "training_stats = train(model=model, optimizer=adam, scheduler=linear_sch, train_dataloader, dev_dataloader, EPOCHS, verbose=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 3 ========\n",
            "  Batch    40  of    177.    Elapsed: 0:00:30.\n",
            "  Batch    80  of    177.    Elapsed: 0:00:58.\n",
            "  Batch   120  of    177.    Elapsed: 0:01:27.\n",
            "  Batch   160  of    177.    Elapsed: 0:01:56.\n",
            "\n",
            "  Average training loss: 1.76\n",
            "  Training epcoh took: 0:02:08\n",
            "  Accuracy: 0.44\n",
            "  Validation Loss: 1.54\n",
            "  Validation took: 0:00:13\n",
            "\n",
            "======== Epoch 2 / 3 ========\n",
            "  Batch    40  of    177.    Elapsed: 0:00:29.\n",
            "  Batch    80  of    177.    Elapsed: 0:00:58.\n",
            "  Batch   120  of    177.    Elapsed: 0:01:26.\n",
            "  Batch   160  of    177.    Elapsed: 0:01:55.\n",
            "\n",
            "  Average training loss: 1.44\n",
            "  Training epcoh took: 0:02:07\n",
            "  Accuracy: 0.47\n",
            "  Validation Loss: 1.49\n",
            "  Validation took: 0:00:13\n",
            "\n",
            "======== Epoch 3 / 3 ========\n",
            "  Batch    40  of    177.    Elapsed: 0:00:29.\n",
            "  Batch    80  of    177.    Elapsed: 0:00:58.\n",
            "  Batch   120  of    177.    Elapsed: 0:01:26.\n",
            "  Batch   160  of    177.    Elapsed: 0:01:55.\n",
            "\n",
            "  Average training loss: 1.32\n",
            "  Training epcoh took: 0:02:07\n",
            "  Accuracy: 0.46\n",
            "  Validation Loss: 1.49\n",
            "  Validation took: 0:00:13\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:59 (h:mm:ss)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CzDOq32qZK_I",
        "colab_type": "code",
        "outputId": "0d3dd6cb-988c-431c-a84e-4664c32f5ceb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 579
        }
      },
      "source": [
        "plot_loss(training_stats)\n",
        "training_stats"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuoAAAGaCAYAAABZrWb6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3gU1f4/8PfMtvSeEEiogQRID02KIr330EVFEFCKCoJ4LVe8X/xdAUVA4SpwRZQiJKEaQPoVpAiE0FuooSQhpCck2d35/RGzsCRAEnYzu+H9eh6fx5yZOfPZTebhvWfPnBEkSZJAREREREQWRZS7ACIiIiIiKolBnYiIiIjIAjGoExERERFZIAZ1IiIiIiILxKBORERERGSBGNSJiIiIiCwQgzoRVVmJiYkICAjAggULKtzH9OnTERAQYMKqqq7Hvd8BAQGYPn16mfpYsGABAgICkJiYaPL6YmJiEBAQgEOHDpm8byIic1DKXQARPT/KE3h37twJX19fM1ZjfXJzc/Gf//wHsbGxSE5OhpubG5o0aYK3334bfn5+Zepj0qRJ2LZtG9avX49GjRqVuo8kSejQoQMyMzOxb98+2NjYmPJlmNWhQ4dw+PBhvPbaa3BycpK7nBISExPRoUMHDB8+HJ9++qnc5RCRhWNQJ6JKM2vWLKOfjx49il9//RWDBw9GkyZNjLa5ubk98/l8fHxw4sQJKBSKCvfxr3/9CzNmzHjmWkzh448/xm+//YaePXuiefPmSElJwa5duxAfH1/moB4ZGYlt27YhOjoaH3/8can7HDx4EDdv3sTgwYNNEtJPnDgBUaycL3APHz6Mb7/9Fv369SsR1Pv06YMePXpApVJVSi1ERM+KQZ2IKk2fPn2MftbpdPj1118RFhZWYtujsrOz4eDgUK7zCYIAjUZT7jofZimhLi8vD1u3bkWbNm3w1VdfGdonTJiAgoKCMvfTpk0bVK9eHZs2bcK0adOgVqtL7BMTEwOgKNSbwrP+DkxFoVA804c2IqLKxjnqRGRx2rdvjxEjRuDMmTMYNWoUmjRpgt69ewMoCuxz587FwIED0aJFCwQFBaFTp06YM2cO8vLyjPopbc70w227d+/GgAEDEBwcjDZt2uDLL7+EVqs16qO0OerFbVlZWfjnP/+Jli1bIjg4GEOGDEF8fHyJ15OWloYPP/wQLVq0QHh4OF599VWcOXMGI0aMQPv27cv0ngiCAEEQSv3gUFrYfhxRFNGvXz+kp6dj165dJbZnZ2fj999/h7+/P0JCQsr1fj9OaXPU9Xo9vv/+e7Rv3x7BwcHo2bMnNm7cWOrxCQkJ+Oyzz9CjRw+Eh4cjNDQU/fv3x9q1a432mz59Or799lsAQIcOHRAQEGD0+3/cHPV79+5hxowZaNu2LYKCgtC2bVvMmDEDaWlpRvsVH3/gwAEsXboUHTt2RFBQELp06YJ169aV6b0oj3PnzmH8+PFo0aIFgoOD0b17dyxevBg6nc5ov9u3b+PDDz9Eu3btEBQUhJYtW2LIkCFGNen1eixbtgy9evVCeHg4IiIi0KVLF/zjH/9AYWGhyWsnItPgiDoRWaRbt27htddeQ9euXdG5c2fk5uYCAJKSkhAVFYXOnTujZ8+eUCqVOHz4MJYsWYKzZ89i6dKlZep/7969WLlyJYYMGYIBAwZg586d+O9//wtnZ2eMGzeuTH2MGjUKbm5uGD9+PNLT0/Hjjz9izJgx2Llzp2H0v6CgACNHjsTZs2fRv39/BAcH4/z58xg5ciScnZ3L/H7Y2Nigb9++iI6OxubNm9GzZ88yH/uo/v37Y9GiRYiJiUHXrl2Ntv3222+4f/8+BgwYAMB07/ej/t//+39Yvnw5mjVrhtdffx2pqan4/PPPUbNmzRL7Hj58GEeOHMHLL78MX19fw7cLH3/8Me7du4exY8cCAAYPHozs7Gxs374dH374IVxdXQE8+d6IrKwsDB06FNeuXcOAAQPQuHFjnD17FqtWrcLBgwexdu3aEt/kzJ07F/fv38fgwYOhVquxatUqTJ8+HbVq1SoxhauiTp48iREjRkCpVGL48OHw8PDA7t27MWfOHJw7d87wrYpWq8XIkSORlJSEYcOGoU6dOsjOzsb58+dx5MgR9OvXDwCwaNEizJ8/H+3atcOQIUOgUCiQmJiIXbt2oaCgwGK+OSKiR0hERDKJjo6W/P39pejoaKP2du3aSf7+/tKaNWtKHJOfny8VFBSUaJ87d67k7+8vxcfHG9pu3Lgh+fv7S/Pnzy/RFhoaKt24ccPQrtfrpR49ekitW7c26veDDz6Q/P39S2375z//adQeGxsr+fv7S6tWrTK0/fLLL5K/v7+0cOFCo32L29u1a1fitZQmKytLevPNN6WgoCCpcePG0m+//Vam4x7n1VdflRo1aiQlJSUZtQ8aNEgKDAyUUlNTJUl69vdbkiTJ399f+uCDDww/JyQkSAEBAdKrr74qabVaQ/upU6ekgIAAyd/f3+h3k5OTU+L8Op1OeuWVV6SIiAij+ubPn1/i+GLFf28HDx40tH399deSv7+/9MsvvxjtW/z7mTt3bonj+/TpI+Xn5xva79y5IwUGBkrvvfdeiXM+qvg9mjFjxhP3Gzx4sNSoUSPp7Nmzhja9Xi9NmjRJ8vf3l/78809JkiTp7Nmzkr+/v/TDDz88sb++fftK3bp1e2p9RGRZOPWFiCySi4sL+vfvX6JdrVYbRv+0Wi0yMjJw7949tGrVCgBKnXpSmg4dOhitKiMIAlq0aIGUlBTk5OSUqY/XX3/d6OcXXngBAHDt2jVD2+7du6FQKPDqq68a7Ttw4EA4OjqW6Tx6vR7vvPMOzp07hy1btuCll17C+++/j02bNhnt98knnyAwMLBMc9YjIyOh0+mwfv16Q1tCQgKOHz+O9u3bG27mNdX7/bCdO3dCkiSMHDnSaM54YGAgWrduXWJ/Ozs7w//n5+cjLS0N6enpaN26NbKzs3H58uVy11Bs+/btcHNzw+DBg43aBw8eDDc3N+zYsaPEMcOGDTOablStWjXUrVsXV69erXAdD0tNTUVcXBzat2+Phg0bGtoFQcBbb71lqBuA4W/o0KFDSE1NfWyfDg4OSEpKwpEjR0xSIxFVDk59ISKLVLNmzcfe+LdixQqsXr0aly5dgl6vN9qWkZFR5v4f5eLiAgBIT0+Hvb19ufsonmqRnp5uaEtMTISXl1eJ/tRqNXx9fZGZmfnU8+zcuRP79u3D7Nmz4evri3nz5mHChAmYNm0atFqtYXrD+fPnERwcXKY56507d4aTkxNiYmIwZswYAEB0dDQAGKa9FDPF+/2wGzduAADq1atXYpufnx/27dtn1JaTk4Nvv/0WW7Zswe3bt0scU5b38HESExMRFBQEpdL4n0OlUok6dergzJkzJY553N/OzZs3K1zHozUBQP369Utsq1evHkRRNLyHPj4+GDduHH744Qe0adMGjRo1wgsvvICuXbsiJCTEcNzkyZMxfvx4DB8+HF5eXmjevDlefvlldOnSpVz3OBBR5WJQJyKLZGtrW2r7jz/+iH//+99o06YNXn31VXh5eUGlUiEpKQnTp0+HJEll6v9Jq388ax9lPb6sim9+bNasGYCikP/tt9/irbfewocffgitVouGDRsiPj4eM2fOLFOfGo0GPXv2xMqVK3Hs2DGEhoZi48aN8Pb2xosvvmjYz1Tv97OYMmUK9uzZg0GDBqFZs2ZwcXGBQqHA3r17sWzZshIfHsytspaaLKv33nsPkZGR2LNnD44cOYKoqCgsXboUo0ePxtSpUwEA4eHh2L59O/bt24dDhw7h0KFD2Lx5MxYtWoSVK1caPqQSkWVhUCciq7Jhwwb4+Phg8eLFRoHpf//7n4xVPZ6Pjw8OHDiAnJwco1H1wsJCJCYmlumhPMWv8+bNm6hevTqAorC+cOFCjBs3Dp988gl8fHzg7++Pvn37lrm2yMhIrFy5EjExMcjIyEBKSgrGjRtn9L6a4/0uHpG+fPkyatWqZbQtISHB6OfMzEzs2bMHffr0weeff2607c8//yzRtyAI5a7lypUr0Gq1RqPqWq0WV69eLXX03NyKp2RdunSpxLbLly9Dr9eXqKtmzZoYMWIERowYgfz8fIwaNQpLlizBG2+8AXd3dwCAvb09unTpgi5dugAo+qbk888/R1RUFEaPHm3mV0VEFWFZwwJERE8hiiIEQTAaydVqtVi8eLGMVT1e+/btodPpsHz5cqP2NWvWICsrq0x9tG3bFkDRaiMPzz/XaDT4+uuv4eTkhMTERHTp0qXEFI4nCQwMRKNGjRAbG4sVK1ZAEIQSa6eb4/1u3749BEHAjz/+aLTU4OnTp0uE7+IPB4+O3CcnJ5dYnhF4MJ+9rFNyOnbsiHv37pXoa82aNbh37x46duxYpn5Myd3dHeHh4di9ezcuXLhgaJckCT/88AMAoFOnTgCKVq15dHlFjUZjmFZU/D7cu3evxHkCAwON9iEiy8MRdSKyKl27dsVXX32FN998E506dUJ2djY2b95croBamQYOHIjVq1fjm2++wfXr1w3LM27duhW1a9cusW57aVq3bo3IyEhERUWhR48e6NOnD7y9vXHjxg1s2LABQFHo+u677+Dn54du3bqVub7IyEj861//wh9//IHmzZuXGKk1x/vt5+eH4cOH45dffsFrr72Gzp07IzU1FStWrEDDhg2N5oU7ODigdevW2LhxI2xsbBAcHIybN2/i119/ha+vr9H9AAAQGhoKAJgzZw569eoFjUaDBg0awN/fv9RaRo8eja1bt+Lzzz/HmTNn0KhRI5w9exZRUVGoW7eu2UaaT506hYULF5ZoVyqVGDNmDD766COMGDECw4cPx7Bhw+Dp6Yndu3dj37596NmzJ1q2bAmgaFrUJ598gs6dO6Nu3bqwt7fHqVOnEBUVhdDQUENg7969O8LCwhASEgIvLy+kpKRgzZo1UKlU6NGjh1leIxE9O8v8l42I6DFGjRoFSZIQFRWFmTNnwtPTE926dcOAAQPQvXt3ucsrQa1W46effsKsWbOwc+dObNmyBSEhIVi2bBk++ugj3L9/v0z9zJw5E82bN8fq1auxdOlSFBYWwsfHB127dsUbb7wBtVqNwYMHY+rUqXB0dESbNm3K1G+vXr0wa9Ys5Ofnl7iJFDDf+/3RRx/Bw8MDa9aswaxZs1CnTh18+umnuHbtWokbOGfPno2vvvoKu3btwrp161CnTh289957UCqV+PDDD432bdKkCd5//32sXr0an3zyCbRaLSZMmPDYoO7o6IhVq1Zh/vz52LVrF2JiYuDu7o4hQ4Zg4sSJ5X4ablnFx8eXumKOWq3GmDFjEBwcjNWrV2P+/PlYtWoVcnNzUbNmTbz//vt44403DPsHBASgU6dOOHz4MDZt2gS9Xo/q1atj7NixRvu98cYb2Lt3L37++WdkZWXB3d0doaGhGDt2rNHKMkRkWQSpMu4EIiIiIzqdDi+88AJCQkIq/NAgIiKq2jhHnYjIzEobNV+9ejUyMzNLXTeciIgI4NQXIiKz+/jjj1FQUIDw8HCo1WrExcVh8+bNqF27NgYNGiR3eUREZKE49YWIyMzWr1+PFStW4OrVq8jNzYW7uzvatm2Ld955Bx4eHnKXR0REFopBnYiIiIjIAnGOOhERERGRBWJQJyIiIiKyQLyZ9G9paTnQ6003C8jd3QGpqdkm64+IHuD1RWQ+vL6IzEMUBbi62pfrGAb1v+n1kkmDenGfRGQevL6IzIfXF5Fl4NQXIiIiIiILxKBORERERGSBGNSJiIiIiCwQgzoRERERkQViUCciIiIiskBc9YWIiIjoCfLycpCdnQGdrlDuUshCKRQqODg4w9a2fMsvPg2DOhEREdFjFBYWICsrDS4uHlCpNBAEQe6SyMJIkoTCwnykp9+FUqmCSqU2Wd+c+kJERET0GFlZ6XBwcIZabcOQTqUSBAFqtQ3s7Z2RnZ1u0r4Z1ImIiIgeQ6stgEZjK3cZZAVsbGxRWFhg0j459cXEDpy+g5i9CbiXmQ83Jw36t/VDy0BvucsiIiKiCtDrdRBFhdxlkBUQRQX0ep1J+2RQN6EDp+/gpy3nUKDVAwBSM/Px05ZzAMCwTkREZKU45YXKwhx/J5z6YkIxexMMIb1YgVaPmL0JMlVERERERNaKQd2EUjPzy9VOREREVBVNmDAGEyaMqfRjqxpOfTEhdydNqaHcxUEjQzVERERExtq0aVqm/dau3Yjq1WuYuRp6GgZ1E+rf1s9ojnoxnU6PtKx8uDoysBMREZF8Pvnkc6Of16xZhaSk25g4cbJRu4uL6zOdZ+7c72Q5tqphUDeh4htGH171pU1wdWz76wZmrYrDB8PCObpOREREsunSpbvRz3v27ERGRnqJ9kfdv38fNjY2ZT6PSqWqUH3PemxVw6BuYi0DvdEy0Bueno5ISckCADSu64avf43H7FVx+GBYBJzsTffEKiIiIiJTmjBhDLKzszFt2j+wYMFcnD9/DsOHv4pRo8bijz/2YOPGdbhw4TwyMzPg6emF7t17YcSIkVAoFEZ9AMC33/4AADh27AgmTRqHmTNn4cqVy1i/PhqZmRkIDg7F1Kn/gK9vTZMcCwDR0WuwevUKpKbehZ+fHyZMeA+LFy8y6tNaMKhXgga+Lnh3YAjmronHnNVxmDo0HI52DOtERETPo+JnrqRm5sPdQp+5kp6ehmnT3kPnzl3RtWsPVKtWVF9s7GbY2tph8ODhsLOzxdGjR7BkyX+Qk5OD8ePfeWq/P/20FKKowLBhryIrKxOrVv2MGTM+xuLFP5nk2HXrojB37iyEhUVg8OChuH37Nj788H04OjrC09Or4m+ITBjUK0lALVe8ExmCb6JO4KvVx/H+0HA42PKrHSIioueJtTxz5e7dFEyf/gl69uxj1P7ZZ/8HjebBFJi+fSMxe/YXWLduLd588y2o1U8eiNRqtfjvf3+CUlkUQZ2cnDFv3hxcvnwJ9erVf6ZjCwsLsWTJIgQGBuObbxYa9qtfvwFmzvyMQZ2erFEdN0wcEIz5USfx1a/HMXVIGOxsGNaJiIisyf6Tt7HvxO0KHZtwKwNanWTUVqDV48fYs/jf8Vvl6qtNSHW0Dq5eoTqexsbGBl279ijR/nBIz83NQUFBIUJDw7FhQwyuXbuKBg38n9hvjx69DQEaAEJDwwAAt27dfGpQf9qx586dQUZGBt5+u5/Rfp06dcX8+V8/sW9LxaBeyYLqumNC/yB8G3MSX/0ajymDw2Bnw18DERHR8+DRkP60drl4enoZhd1ily8nYPHiRTh27C/k5OQYbcvJyX5qv8VTaIo5OjoBALKysp752Dt3ij48PTpnXalUonp183ygMTcmRBmE+Hng7b7B+G7dScxdexyTB4XBVsNfBRERkTVoHVzxkeypC/eX+swVdycNPhge8aylmczDI+fFsrKyMHHiGNjZOWDUqHHw8fGFWq3GhQvnsGjRAuj1+lJ6MiaKilLbJenpH1Se5VhrxSeTyiSsgQfG9QnElVtZmLc2HvkFOrlLIiIiIjPr39YPaqVx/FIrRfRv6ydTRWUXF3cUGRkZ+Oijf2LQoKFo3fpFNGvWwjCyLTdv76IPT4mJN4zatVotbt+u2FQluTGoy6hJgBfG9G6MizczMC8qHvmFDOtERERVWctAb7zWrSHcnYqeq+LupMFr3Rpa1I2kjyOKRbHx4RHswsJCrFu3Vq6SjDRs2BjOzs7YuHEdtFqtoX379q3IysqUsbKK43wLmTVvVA06vYQlm85gQfQJTBoQArWq9K92iIiIyPoVP3PF2gQHh8DR0QkzZ36GyMjBEAQB27bFwlJmnqhUKrzxxhjMnTsb7777Ntq164Dbt29jy5ZN8PHxhSAIcpdYbhxRtwAtA73xRo9GOHs1Dd+uO4lCLUfWiYiIyLI4O7tg1qy5cHf3wOLFi7Bq1S9o2rQF3n57ktylGQwYMBjvvvs+7ty5je++m4f4+Dj8+99fw8HBEWq19T0dXpCq8gz8ckhNzYZeb7q34uEnk5bV/+JvYdmWcwjxc8eE/sFQKvg5iqg0Fbm+iKhseH0Zu3PnGry9a8tdBj0DvV6Pnj07oW3bdvjgg4/Neq4n/b2IogB3d4dy9cckaEFeCq2BEV0CcCIhFYvWn4JW9/S7p4mIiIioSH5+yRV1tm79DZmZGQgPbyJDRc+Gc9QtTLtwH+j1ElZsv4AfNp7G2D6BUIj8PEVERET0NCdOHMeiRQvw8svt4eTkjAsXzuG33zaiXj0/tGvXUe7yyo1B3QJ1aOILnU6P1bsuQdx0Bm/2asywTkRERPQUNWr4wMPDE1FRvyIzMwNOTs7o2rUHxo2bAJXK+p4Gz6BuoTo3rwWdXsLaPQlQiAJG9WgMUbS+u5WJiIiIKouPjy9mzZordxkmw6Buwbq9UBtavYR1/7sMURQwsnsjiFa4tBARERERlR+DuoXr1aoOdDo9Nu6/CoUo4tWuAQzrRERERM8BBnUr0KdNXej0En47cA0KUcArnf2tctF+IiIiIio7BnUrIAgC+r9UDzq9hK2HrkMhChjasQHDOhEREVEVxqBuJQRBwMCX/aDTSdh+5AYUCgGD2tVnWCciIiKqohjUrYggCBjSoT70egnbDt+AQhQxoG09hnUiIiKiKohB3coIgoBhnRpAp9cj9mDRnPV+L9WTuywiIiIiMjE+RccKCYKAV7oE4MWQ6tj051Vs3H9F7pKIiIjoORQbuwlt2jTF7du3DG2Rkb0wc+ZnFTr2WR07dgRt2jTFsWNHTNannBjUrZQoCHitW0O0CvLG+j+u4LcDV+UuiYiIiCzctGnvoWPHNsjLy3vsPpMnT0CXLm2Rn59fiZWVz44d27BmzUq5yzA7Tn2xYqIg4I3ujaDXS4jeexkKUUTXFrXkLouIiIgsVKdOXfDnn39g37696NSpa4ntaWn3cPToX+jcuRs0Gk2FzrFyZTRE0bxjwTt3/o6LFy9g0KBhRu1hYRHYuXM/VCqVWc9fWTiibuVEUcCono3QrKEX1uy+hO1HbshdEhEREVmoF198Gba2dtixY1up23ft2gGdTofOnUuG+LJSq9VQKuUZCxZFERqNxuwfFCoLR9SrAIUo4s1ejaHXS1i14yKUooB2Eb5yl0VEREQWxsbGBi++2Ba7d+9AZmYmnJycjLbv2LEN7u7uqFmzNubM+TeOHj2MpKQk2NjYICKiKcaPfwfVq9d44jkiI3shPLwJPvroM0Pb5csJ+Oab2Th16iScnZ3Rp09/eHh4ljj2jz/2YOPGdbhw4TwyMzPg6emF7t17YcSIkVAoFACACRPG4PjxYwCANm2aAgC8vasjKmoTjh07gkmTxmH+/P8gIqKpod+dO3/HL78sw7VrV2FnZ4/WrV/EW29NgouLi2GfCRPGIDs7G59++jm+/noWzp49DUdHJwwcOATDh79WvjfaRBjUqwilQsTYPoFYuO4Ufv79AkRRQNswH7nLIiIiokccvnMMGxO2Ii0/Ha4aF/T264rm3hGVdv5Onbri99+3YM+enejdu5+h/c6d2zh16gQiI4fg7NnTOHXqBDp27AJPTy/cvn0L69dHY+LEsfjll7WwsbEp8/lSU+9i0qRx0Ov1eOWV12BjY4uNG9eVOrUmNnYzbG3tMHjwcNjZ2eLo0SNYsuQ/yMnJwfjx7wAAXnvtDeTl5SEp6TYmTpwMALC1tXvs+WNjN+GLL2YgMDAYb701CcnJSYiO/hVnz57G4sXLjerIzMzAlCmT0K5dB3To0Bm7d+/AokULUK9efbRs2brMr9lUGNSrEKVCxFt9g/DdupNYvvU8FKKINiHV5S6LiIiI/nb4zjGsPBeNQn0hACAtPx0rz0UDQKWF9WbNWsDFxRU7dmwzCuo7dmyDJEno1KkL/Pzqo127jkbHtW79EsaNG4k9e3aia9ceZT7fihU/ISMjHUuW/IyAgIYAgG7demLo0H4l9v3ss/+DRvPgQ0DfvpGYPfsLrFu3Fm+++RbUajWaNXsBMTFrkZGRji5duj/x3FqtFosWLUD9+v5YsOB7qNVqAEBAQEN89tlH2LRpHSIjhxj2T05Owj//+X+G+fs9e/ZBZGRP/PbbBgZ1enYqpYjx/YIwP/okfow9C4UooGWQt9xlERERVRmHbh/Fgdt/VejYKxnXoZW0Rm2F+kKsOBuFP28dLldfLas3Q4vqTcpdg1KpRPv2HbF+fTTu3r0LDw8PAMCOHb/D17cmGjcOMtpfq9UiJycbvr414eDgiAsXzpUrqB84sB/BwaGGkA4Arq6u6NSpG9atW2u078MhPTc3BwUFhQgNDceGDTG4du0qGjTwL9drPXfuDNLS7hlCfrH27Tvhu+/m4c8/9xsFdQcHB3Ts2MXws0qlQqNGgbh162a5zmsqDOpVkEqpwMT+wZgXdQJLfjsDURTQonE1ucsiIiJ67j0a0p/Wbi6dOnVFTMxa7Nr1OwYNGoarV6/g0qULGDnyTQBAfv59/PzzMsTGbkJKSjIkSTIcm52dXa5zJSXdQXBwaIn2WrVql2i7fDkBixcvwrFjfyEnJ8doW05O+c4LFE3nKe1coijC17cmkpJuG7V7eVUr8cR3R0cnJCRcKve5TUHWoJ6cnIzly5cjPj4ep06dQm5uLpYvX44WLVo89diAgIDHbmvVqhV+/PFHU5ZqddQqBSYNCMHctfFYvOkMFKKApg295C6LiIjI6rWo3qRCI9kA8PH+L5CWn16i3VXjgncjxj1raWUWHByK6tV9sH37VgwaNAzbt28FAMOUj7lzZyM2dhMGDhyKoKBgODg4ABDw2Wf/MArtppSVlYWJE8fAzs4Bo0aNg4+PL9RqNS5cOIdFixZAr9eb5bwPE0VFqe3mes1PI2tQv3LlChYvXozatWsjICAAcXFxZT521qxZJdpOnTqF5cuXo3Xryp9DZIk0agXeiQzB3DXx+H7jaShEAeH+Je+wJiIiosrR26+r0Rx1AFCJKvT2q/hyiBXVsWNn/Pzzj0hMvIGdO39HQEAjw8hz8Tz0iRPfM+yfn59f7tF0AKhWzRuJiSWXj75+/ZrRz3FxR5GRkYGZM/hh+GcAACAASURBVGcjLOzBfP3Sn1wqlNJWkrd3dcO5Hu5TkiQkJt5A3bp+ZepHLrIuMhkYGIiDBw/i999/x+jRo8t1bJ8+fUr8l5ubC0EQ0LNnTzNVbH1sNUq8NygUtb0dsXD9KRy/dFfukoiIiJ5bzb0jMKzhALhqipYFdNW4YFjDAZW66kuxzp27AQC+/XYuEhNvGK2dXtrIcnT0r9DpdOU+T8uWrXHyZDzOnz9naEtLS8P27VuM9ite+/zh0evCwsIS89gBwNbWtkwfGho2bAxXVzesXx+FwsIHH452796JlJRktGpl2YO7so6oF32NYhoFBQX4/fff0axZM3h78+bJh9lqlJg8KBRzVh/HwnUnMXFACILructdFhER0XOpuXeELMH8UXXr1kP9+v7Yt+9/EEURHTo8uImyVas22LYtFvb2DqhTpy5Onz6JI0cOw9nZudznGTbsNWzbFovJk8cjMnIINBobbNy4DtWqVUd29kXDfsHBIXB0dMLMmZ8hMnIwBEHAtm2xKG3WSUBAQ/z++xYsWPA1GjZsDFtbO7Rp81KJ/ZRKJd56ayK++GIGJk4ci44dOyM5OQlRUb+iXj0/9OpVcuUZS1I1HtsEYO/evcjMzETv3r3lLsUi2dmoMHlwGGq422NB9EmcvnpP7pKIiIhIZsWj6OHhTQyrvwDAO++8jy5dumP79i349ttvcPfuXXzzzXdPXK/8cTw8PDB//veoW9cPP/+8DGvXrkLXrt0xcOAQo/2cnV0wa9ZcuLt7YPHiRVi16hc0bdoCb789qUSfffoMQJcu3RAbuxkzZnyMb76Z/djzd+/eC599NhP5+ffx3XfzEBu7CZ06dcW8ef8pdS13SyJIcs2Of8SOHTswfvz4Mt9M+qhJkyZh9+7d2L9/f4mnbJVFamo29HrTvRWeno5ISckyWX+mkp1XiFkrjyE5LQ/vDgxFw9qucpdEVG6Wen0RVQW8vozduXMN3t4lVychKs2T/l5EUYC7e/lmk1SJ5Rmzs7OxZ88etG3btkIhHUC537iy8PR0NHmfz8oTwP8b/yL+sWg/5kWfwIw3WyKQ02DIClni9UVUVfD6eiA5WYRSWWUmIJCZiaJo0uunSgT1bdu2IT8/H7169apwH8/LiHqxyQND8OXKOPxz8QFMGRSG+r7ln3NGJBdLv76IrBmvL2N6vR5arfmXBaSqQa/XP/b6qciIepX4iLhp0yY4OjqiXbt2cpdiNZwdNJg6NBwu9mp8veY4Lt/KlLskIiIiInqI1Qf15ORkHDp0CJ07dzZ6NCw9natjUVh3tFPhq1+P4+odhnUiIiIiS2EVQf369eu4fv16qdtiY2Oh1+ufadrL88zNyQZTh4bDTqPEV6uP43oSv+4kIiIisgSyz1FfuHAhACAhIQEAsGHDBhw9ehROTk545ZVXAACvv/46AGDXrl0ljt+4cSO8vLwqtFIMFfFwtsW0YeH4cuUxzFl9HNOGhsPXy/Q31xIRERFR2cke1OfNm2f0c3R0NADAx8fHENQf5/Llyzh9+jRGjhxpeJoVVYyniy2mDg3HlyuOYfbqOEwbFgEfD3u5yyIiIiJ6blnMOupye95WfXmcO/dy8eWKY5AAfDAsHNXdGdbJ8ljr9UVkDXh9Gbtz5xqqVasFQRDkLoUsnCRJSEq6btJ11DkMTUa83ewwdWg4IEmYvSoOSWm5cpdEREQkG4VCicLCArnLICtQWFgAhcK0k1UY1KmEGh72eH9oOLQ6CbNWxiE5PU/ukoiIiGTh4OCC9PQUFBTkg5MQqDSSJKGgIB/p6SlwcHAxad+c+vI3Tn0p6XpSFmavioONWoEPhkfAw9lW7pKIAFSN64vIUvH6KikvLwfZ2enQ6bRyl0IWSqFQwsHBBba2j58yXJGpLwzqf2NQL921O0Vh3c5GienDI+DmZCN3SURV5voiskS8vojMg3PUyeRqeztiypAw5NwvxKxVcUjLype7JCIiIqLnAoM6PVXd6k54b1AYMnIKMHtVHDKyGdaJiIiIzI1Bncqkvo8z3hsYirSsfMxefRyZObwDnoiIiMicGNSpzPxruuDdgSG4m56HOavjkJXLsE5ERERkLgzqVC4BtVwxKTIESWl5+Gr1cWTnFcpdEhEREVGVxKBO5da4jhsm9g/GrdQcfPXrceTeZ1gnIiIiMjUGdaqQoHruGN8vGInJ2fh6TTzy8rm2LBEREZEpMahThYXW98DbfYNw7U4W5jKsExEREZkUgzo9k3B/T4ztHYjLtzIxb2088gt0cpdEREREVCUwqNMza9rQC2N6N8bFmxmYFxWP/EKGdSIiIqJnxaBOJtG8UTWM7tEY56+n49voEyjUMqwTERERPQsGdTKZlkHeGNm9Ec5cTcOCmJMo1OrlLomIiIjIajGok0m1CamO17o1xKnL97Bw3UlodQzrRERERBXBoE4m91JoDYzo7I/4hFT8Z8NphnUiIiKiCmBQJ7NoF+GLYR0b4NiFFPyw6Qx0eoZ1IiIiovJQyl0AVV0dm9aETi/h112XoBAFvNmzMURRkLssIiIiIqvAoE5m1aV5Lej0EqL2JEAUBIzq0YhhnYiIiKgMGNTJ7Lq/UBs6nR7r/rgChSjg9e4NIQoM60RERERPwqBOlaJX67rQ6SVs3H8VCoWAEV0CGNaJiIiInoBBnSpNnzZFYf23A9cgigJe6eQPgWGdiIiIqFQM6lRpBEFA/5fqQaeTsPXwdShEAUM7NGBYJyIiIioFgzpVKkEQMLCdH7R6PXYcSYRSFDGwnR/DOhEREdEjGNSp0glC0Ui6Xv/3yLqiaKSdYZ2IiIjoAQZ1koUgCBjWyd8wZ10hCuj7Yj25yyIiIiKyGAzqJBtRKFr9xbAajCigV+u6cpdFREREZBEY1ElWoiDg9a4NodNJReusK0R0f6G23GURERERyY5BnWQnikVPLNVLRU8wVYoCOjevJXdZRERERLJiUCeLIIoCRvdsBJ1ewupdlyCKAjo2rSl3WURERESyYVAni6EQRYzp1Rh6vYSVOy5CoRDRLtxH7rKIiIiIZCHKXQDRw5QKEeP6BCLUzx0/bzuP/8XfkrskIiIiIlkwqJPFUSpEvN0vGEH13PDTlnPYf/K23CURERERVToGdbJIKqWICf2C0aiOK/7721kcOH1H7pKIiIiIKhWDOlkstUqBiQNCEFDLBUs2n8Hhs0lyl0RERERUaRjUyaJpVAq8ExmKBj7O+GHjGRw5lyx3SURERESVgkGdLJ5GrcA7A0NRr4YTvt94GnEXUuQuiYiIiMjsGNTJKthqlHhvUChqVXPEwvWnEH/prtwlEREREZkVgzpZDVuNElMGh8LXywHfrTuJU5dT5S6JiIiIyGwY1Mmq2NmoMGVwGGq422NBzEmcuXpP7pKIiIiIzIJBnayOg60KU4aEoZqrLeZHncC5a2lyl0RERERkcgzqZJUc7dR4f0g4PFxsMS/qBC7cSJe7JCIiIiKTYlAnq+Vkr8bUIWFwddRg7tp4XLqZIXdJRERERCbDoE5WzdlBg6lDw+Fsr8bcNcdx+Vam3CURERERmQSDOlk9V0cNpg0Nh72NCl//ehzX7mTJXRIRERHRM2NQpyrBzckG04aFw1ajxJzVcbiexLBORERE1o1BnaoMD2dbTB0WDrVKgTmrjyMxJVvukoiIiIgqjEGdqhQvF1tMGxYOpULAnFVxuHU3R+6SiIiIiCqEQZ2qnGqudpg6NByCIGD2qjjcTmVYJyIiIuvDoE5VUnV3e7w/NBx6ScLsVXFISsuVuyQiIiKicmFQpyrLx8MeU4eGQ6srCusp6Xlyl0RERERUZgzqVKX5ejrg/SFhyC/QYdbKONzNYFgnIiIi6yBrUE9OTsacOXMwYsQIhIeHIyAgAIcOHSrz8Xq9Hr/88gt69eqFkJAQvPDCCxg1ahSuX79uxqrJ2tSq5ogpQ8KQm6/F7FVxuJd5X+6SiIiIiJ5K1qB+5coVLF68GElJSQgICCj38dOmTcOcOXPQokULfPLJJxg7diycnJyQnp5uhmrJmtXxdsKUwWHIzivE7FVxSMvKl7skIiIioidSynnywMBAHDx4EK6urtixYwfGjx9f5mM3b96MrVu3YsWKFQgNDTVjlVRV1KvhhPcGheGrX49jzuo4TBsWAWd7tdxlEREREZVK1hF1BwcHuLq6VujYn376CR07dkRoaCi0Wi3y8jj3mJ6uvo8z3hsYitTM+5i9Kg6ZuQVyl0RERERUKqu8mTQ7OxsnT55EQEAAPv30U4SHhyMsLAw9e/bEvn375C6PLJx/TRe8ExmKu+l5mLPqOLLzCuUuiYiIiKgEqwzq169fhyRJWLZsGQ4ePIjPPvsMX375JQBg7NixOHHihMwVkqVrVNsVEyNDcOdeLuasjkPOfYZ1IiIisiyyzlGvqNzcoofX5OTkYP369ahevToA4MUXX0THjh3x/fff47vvvitXn+7uDiav09PT0eR9kum87OkIR0cb/N9/D2Ne9En8a2wrONiq5C6LyojXF5H58PoisgxWGdQ1Gg0AICIiwhDSAcDd3R2tWrXCsWPHyt1namo29HrJZDV6ejoiJSXLZP2RedRyt8Pb/YLwXcxJfLRwH6YMDoOtxiovi+cKry8i8+H1RWQeoiiUe2DYKqe+eHl5AQA8PDxKbHN3d0dmZmZll0RWLKy+B97qG4Rrd7Iwd2087hdo5S6JiIiIyDqDerVq1eDh4YGkpKQS25KSkiq8kgw9vyL8PTG2dyAu38zEN2tPIL9AJ3dJRERE9JyziqB+/fr1Ek8b7dq1K+Li4pCQkGBoS0xMxP79+9GqVavKLpGqgKYNvfBmr8a4mJiO+dEnkF/IsE5ERETyESRJMt3E7ApYuHAhACAhIQGbN2/GgAED4OvrCycnJ7zyyisAgPbt2wMAdu3aZTguOTkZ/fr1gyAIGDFiBBQKBX755RdkZWUhJiYGtWvXLlcdnKNOxQ6cuoMlm8+gcR1XTIoMgUqpkLskegSvLyLz4fVFZB4VmaMue1APCAgotd3Hx8cQzEsL6gBw9epV/Pvf/8bhw4chSRIiIiIwbdq0x/b5JAzq9LA/TtzCj7HnEFzPHRP6B0OltIovn54bvL6IzIfXF5F5WGVQtxQM6vSovcdv4qet5xFW3wNv9wuCUsGwbil4fRGZD68vIvN4blZ9IaoMbcN88Epnfxy/dBf/2XAaWp1e7pKIiIjoOcKgTvQE7SN8MbRjAxy7kIIfNp2BTs+wTkRERJWDT3YheopOTWtCp5OwZvclKEUBo3s2higKcpdFREREVRyDOlEZdG1RCzq9HtF7L0MUBbzRvRHDOhEREZkVgzpRGfVoWQc6vYT1f1yBKAp4vVtDiALDOhEREZkHgzpROfRuXRc6nYRNf16FUhQwoksABIZ1IiIiMgMGdaJy6vtiXej0EmIPXoMoChjeyZ9hnYiIiEyOQZ2onARBwIC29aDT67Ht8A0oRBFDOtRnWCciIiKTYlAnqgBBEDCoXX3o9BK2H7kBhULAwJf9GNaJiIjIZBjUiSpIEAQM7dAAOr2ErYeuQyEK6P9SPYZ1IiIiMgkGdaJnIAhFc9R1Ogm/HbgGpUJEnzZ15S6LiIiIqgAGdaJnJAoCXu0aAL1ewoZ9V6AQBfRsVUfusoiIiMjKMagTmYAoFK2rrtPrEfO/y1AoBHRrUVvusoiIiMiKMagTmYgoCnijRyPo9BLW7k6AQhDQuXktucsiIiIiK8WgTmRCClHEm70aQ6eXsHrXJSgUIjo08ZW7LCIiIrJCoik60Wq12LZtG9asWYOUlBRTdElktRSiiLG9AxHewAMrtl/AnribcpdEREREVqjcI+qzZs3CoUOHEB0dDQCQJAkjR47EkSNHIEkSXFxcsGbNGtSqxa/86fmlVIgY1ycI3607ieXbzkMUBbwUWkPusoiIiMiKlHtE/Y8//kDTpk0NP+/atQt//fUXRo0aha+++goA8MMPP5iuQiIrpVKKGN8vCEF13fDTlnPYf/K23CURERGRFSn3iPqdO3dQu/aD1Sx2794NX19fvP/++wCAixcvYtOmTaarkMiKqZQKTOgfjPnRJ/Df385CIQp4IdBb7rKIiIjICpR7RL2wsBBK5YN8f+jQIbRq1crwc82aNTlPneghapUCEweEIKCWCxZvPoPDZ5PkLomIiIisQLmDure3N+Li4gAUjZ7fuHEDzZo1M2xPTU2FnZ2d6SokqgI0KgUmRYagvo8zfth4BkfPJ8tdEhEREVm4ck996dGjBxYuXIh79+7h4sWLcHBwQNu2bQ3bz549yxtJiUpho1bi3YGh+HrNcfxnw2m83U9AeANPucsiIiIiC1XuEfWxY8eiX79+OH78OARBwJdffgknJycAQFZWFnbt2oWWLVuavFCiqsBWo8R7A8NQq5ojFq47hRMJd+UuiYiIiCyUIEmSZKrO9Ho9cnJyYGNjA5VKZapuK0Vqajb0epO9FfD0dERKSpbJ+qOqJfd+IWavOo6bd3MwKTIYQXXd5S7JqvD6IjIfXl9E5iGKAtzdHcp3jCkL0Gq1cHR0tLqQTlTZ7GxUmDIkDNXd7bAg+iTOXL0nd0lERERkYcod1Pfu3YsFCxYYta1YsQIREREICwvDlClTUFhYaLICiaoqB1sV3h8SBi9XW8yPOoHz19PkLomIiIgsSLmD+tKlS3H58mXDzwkJCfjiiy/g5eWFVq1aITY2FitWrDBpkURVlaOdGlOHhMPd2QbfrD2BCzfS5S6JiIiILES5g/rly5cRFBRk+Dk2NhYajQZRUVFYsmQJunfvjvXr15u0SKKqzMlejalDw+HiqMHctfFIuJkhd0lERERkAcod1DMyMuDq6mr4+c8//8QLL7wAB4eiyfHNmzdHYmKi6Sokeg64OGgwbWg4nO3U+HrNcVy5nSl3SURERCSzcgd1V1dX3Lp1CwCQnZ2NkydPomnTpobtWq0WOp3OdBUSPSdcHTWYNiwc9jYqfLX6OK7d4aoLREREz7NyB/WwsDCsXr0aW7duxRdffAGdToeXXnrJsP3atWvw8vIyaZFEzws3JxtMGxoOW40Cc1bH4XoSwzoREdHzqtxBfdKkSdDr9Xj33XcRExODvn37on79+gAASZKwY8cOREREmLxQoueFh4stpg6LgFqlwJzVx5GYki13SURERCSDCj3wKD09HceOHYOjoyOaNWtmaM/IyMD69evRokULNGzY0KSFmhsfeESWJuleLv698hgkvYRpwyJQw8Ne7pIsBq8vIvPh9UVkHhV54JFJn0xqzUwV1A/fOYaNCVuRnp8OF40Levt1RXNvfsNAFXM7NQdfroyDAOCD4RHwdrOTuySLwCBBZD68vojMo1KD+vXr17Fz507cuHEDAFCzZk106NABtWrVqkh3sjNFUD985xhWnotGof7BA59UogrDGg5gWKcKu3k3B7NWHoNSIeKDYeHwcmVYZ5AgMh9eX0TmUWlB/ZtvvsHixYtLrO4iiiLGjh2Ld955p7xdys4UQf3j/V8gLb/kA2vslLYYHTQC1ew94ax2giAIz3Qeev4kJmdj1qo4qFUipg+LgIeLrdwlyYpBgsh8eH0RmUelBPWoqCh8/PHHCA8Px+jRo9GgQQMAwMWLF7F06VLExcVh5syZ6N+/f7kKkZspgvr4XdOeuo9aoUY1Ww942Xn+/Z8Hqv39/7ZKm2c6P1Vt15OyMHtVHGw1SnwwLALuzs/v3wuDBJH58PoiMo9KCer9+/eHSqXCihUroFQqjbZptVoMHz4chYWFiImJKVchcjPniLqzxgmvNhqM5NwUJOfeRVJuCpJzU5B6Pw0SHpzTSe34d2gvCvLFAd7Dxg0KUfFMtVHVcOV2JuasPg5HWxU+GB4BV0eN3CXJgkGCyHx4fRGZR0WCuvLpuxhLSEjA5MmTS4R0AFAqlejevTu+/vrr8nZbJfT261rqHPW+ft3R0K0BGro1MNq/UFeIlLxUJOfdRXJOCpLyigJ8fMppZBfmGPYTBREetm5Fwd3W86Ew7wUntQOn0jxH6lZ3wuTBofhq9XHMWnkMHwyPgIvD8xnWiYiIqrpyB3WVSoXc3NzHbs/JyYFKpXqmoqxV8Q2jZV31RaVQoYaDN2o4eAOexttyCnORnJvy9+j7g1H4s/cuQqvXGvazUdg8NH3mwTQaLztPaBRqs71Wko9fDWe8NygUX/8aj9mr4jBtWASc7fm7JiIiqmrKPfVl5MiRuHLlCqKiouDh4WG0LTU1FQMGDICfnx+WLl1q0kLNzVrWUddLeqTdT38Q3vNSkJSTguS8u0i7n240lcZF42w8D97WA9XsvOBu6wpRKPezrsjCnL+ehrlr4+HpbIupw8LhZPf8hHV+NU9kPry+iMyjUuao//XXX3j99ddhb2+PAQMGGJ5KeunSJcTExCAnJwfLli1D06ZNy1WI3KwlqD9Jga4QKXkPRt+Lw3xSbgrytHmG/ZSCAh627kaj78Vh3kFlz6k0VuTs1Xv4JuoEqrnaYdqwcDjYPh/fZjFIEJkPry8i86i05Rl37dqFf/3rX7h9+7ZRe40aNfDpp5/i5ZdfLm+XsqsKQf1xJElCdmGO0RSa4mk1KXmp0EkPltm0Vdqimp1niZtaPW09oFY8HyHQ2py+cg/zok6ghocdpg4Nh71N1f89WdL1RVTV8PoiMo9KfeCRXq/HqVOnkJiYCKDogUeBgYFYs2YNli9fjtjY2Ip0K5uqHNSfRKfX4d799KIpNI/Mh0/PzzDa11XjYhiFfzjMu9q4cCqNzE4kpOLbmBOo6eWAKYPDYWdT7ttPrIq1XF9E1ojXF5F5VMqqLw9OJiIkJAQhISFG7Wlpabhy5UpFu6VKphAV8LRzh6edOwLdGxptu6/NR0re3RI3tR6+cxT3dfmG/VSiEp6GteE9HgrxnrBX8SmalSHEzx1v9w3Gd+tOYu6a45g8OAy2mqod1omIiKo6/ktOj2Wj1KCmow9qOvoYtUuShMyCbKMpNMl5KbiVcxsn7p6GXtIb9nVQ2T+YQmPrCS/7optaPe08oBL552dKYQ08MK5PEBatP4W5a+MxeVAobNR8j4mIiKwV/xWnchMEAc4aRzhrHNHAtZ7RNp1eh7v37xmNwifnpuBs6nkcLDjyoA8IcLNxLXU+vLPGiVNpKqhJgCfG9gnE9xtOY97aE3h3YCg0aj4si4iIyBoxqJNJKUSFIXwHP7ItT3u/xNNZk3NTcCnjCgp0BYb91KIKng+tCf9wmLdV2lbuC7JCzRp6QafXY/GmM5gffQLvRIZArWJYJyIisjYM6lRpbJU2qO1UE7Wdahq1S5KEjIJMwyh88Uj89aybiEs+abQ2vKPawejprMUh3t3WDUpOpTF4obE39HoJSzefxYKYk5g0IBgqJcM6ERGRNSlTsvnxxx/L3OGxY8cqXAw9nwRBgIvGGS4aZ/i71jfaptVrcTcvFUl/T6EpDvMn755BVmG2YT9REOFh42aYQuP10Ci8s9rpuVwbvlVQdeh0En7ccg7frTuF8f2CoVJyShEREZG1KFNQ//LLL8vV6fMYisg8lKIS3vbV4G1frcS23MJc4wD/9wo159MSUKgvNOynUagfCu6eqGbrYbip1UZpU5kvp9K9GFoDOknC8q3nsWj9KbzdLwhKBcM6ERGRNShTUF++fLm56yAqNzuVHeo610Jd51pG7XpJj/T8DKMnsybnpuBKxjUcTYo3mkrjrHYyTKF5OMy727hCIVaNqSIvh/lAp5OwYvsFfL/hNMb2CWRYJyIisgIVfuBRVfO8PvDoeVOoK0RKXmqJ+fDJuSnI0eYa9lMICnjYuj8U4j1Qzc4L1ew84aCyt8pvjbb/dQOrdl5Es4ZeGNO7MRSi9YZ1Xl9E5sPri8g8KvWBR0TWSKVQoYaDN2o4eJfYll2Y83eAN54Pfzb1PLSSzrCfrdKmaB78Ize1etl5QK1QV+bLKZdOzWpCp5ewZvclKEQBo3s2hiha3wcOIiKi54WsQT05ORnLly9HfHw8Tp06hdzcXCxfvhwtWrR46rHTp0/HunXrSrSHhoZizZo15iiXqjgHlT0cnO1Rz7mOUbte0uPe/XSjJSWTc+/iUvpl/JVkfPO0q8bFaCpN8XQaNxsXi1gbvmuLWtDp9YjeexkKUcDIHo0gWuG3A0RERM8DWYP6lStXsHjxYtSuXRsBAQGIi4sr1/G2traYMWOGUZubm5spSyQqWlHG1g0etm4IdA8w2lagKyiaOpN3F0k5D57S+ldSHPK09w37KUUlPG3dH8yDt/VANfuiIO+gsq/U19OjZR3odBLW77sCURTwWreGDOtEREQWSNagHhgYiIMHD8LV1RU7duzA+PHjy3W8UqlEnz59zFQd0dOpFWr4OtaAr2MNo3ZJkpBdmGMYhS+eC38nJxmn7p6F7qGpNPZKu79H341vavW0dYdKoTJL3b3b1IVWL2Hzn1ehUIgY0dnfKufdExERVWWyBnUHh/JNqC+NTqdDXl6eSfoiMhVBEOCodoCj2gH1XeoabdPpdUi9n2Y0Dz459y7O3buIQ3eOPugDAtxsXEqsC1/NzhMuGudnnkrT78W60On12HLwOhSCgGGdGjCsExERWRCrvpk0JycHTZo0QV5eHlxcXNC3b19MnjwZGo1G7tKIHkshKv5+MJMHgEZG2+5r7yM5765hacniMH/w9lXk6woM+6lEleHhTtVsjR/yZKeyLVMdgiAgsq0fdDoJv/91AwqFgMHt6zOsExERWQirDeqenp4YPXo0GjVqBL1ej927d2PZsmVISEjAkiVL5C6PqEJslDao5eiLWo6+Ru2SJCGzIKvEVJqbWbcQn3IKeklv2NdBZf/IuvBFo/Aetu5QisaXvCAUhXO9/u+wLgqIfNmPYZ2IiMgCWG1QnzJlitHPPXv2RLVq1bB06VLs378frVu3Lld/5V3Xsiw8OWAwAgAAIABJREFUPR1N3ic9v7zghPrwKdGu1WmRnHMXt7KScCsrGbeyknA7Kxln0s7jwO2/DPsJggAvew/UcKyG6o5eqOFYDTUcvVDD0RsTh4RDpVFiy59X4eRog1e6NSpxHkvD64vIfHh9EVkGqw3qpXnjjTewdOlSHDhwoNxBnQ88Imumgj1qq+uhtns9wP1Be542z2gaTVJuCpKz7uJ00nkU6AsN+6kVanjZesCniQbRZy/iSs5pdAtvDC87T9gqbWR4RU/G64vIfHh9EZnHc//AIw8PD6hUKmRkZMhdCpFFsFXaorZTTdR2qmnUrpf0yMjPNHoya1JeCu7bp0BVIw0ndQk4eWQnAMBJ7WiYQvNgOo0nPGzcoBAVcrwsIiKi50KVCup37txBYWEh11InegpREOFq4wJXGxc0dGtgtC1fW4jvtx7GiZvXEdbYBs7uhUjOvYv4lNPILswx6sPD1u3vdeE9HwrzXnBSO3CeOxER0TOyiqB+/fp1AECtWrUAAPn5+SgsLCyxJOPChQsBAG3atKncAomqEI1ShfHdW2LxJmcc3p+MIR0aYESzohH5nMJcw5NZkww3tabg3L2LKNRrDX3YKGweWhfe+EmtGoVarpdGRERkVWQP6sXhOiEhAQCwYcMGHD16FE5OTnjllVcAAK+//joAYNeuXQCAlJQU9OvXDz179kS9evUMq74cOHAA3f9/e/ceHFV9/3/8tZsbgdxvG5pAgASTbEASELLIeAOcsVaFKhZRsF5Ka8F+v9ivdqqddqbVTh1Fq1KxFvr9VmaqtEVrLL96x6kXTEAIAbIJl3ANmM2dEHLPnt8fuUhIUCDZ7J7N89HONDk5l3fCvLuvc/LZd268UTNnzhz+bwTwIwFWq35wk12dbkMbPzygAKtF82Yka0zQaE2MTNHEyJQ++7sNt+paTvUuoekJ82WnjugL1y4Z+ur9H1Ehkf0m0iSExis2NPobZ8Nvq9ipt8reUX1rvaJConRL6g2alTjdIz8DAAC8zWIYxtC9g/ISpKenD7g9KSmpN5jPnTtX0ldBvaGhQY8//riKiopUWVkpt9utCRMm6Lvf/a7uvvtuBQRc/LpZ3kwK9NfR6dbaf+7VroPVuvuGdF2b3X/qzDdp62xXVXN1n/XwPW9sbepo7t0v0BKguNDYPk/fe8J8WNAYbXcV6tXS19V+1ptgg6xBujPjNsI6MIR4/QI841LeTOr1oO4rCOrAwNo73Hrxn3u0u6xG9347Q1dN+9aQnNcwDJ1pb+qzhKbriXy1qpuq1WF09u47OjBUrZ1t6jxrW4+woDG6M2ORLOoaQWlR19r4no8tsqjrv93/6f5Ysny1T59tOuc469ecW91r8XvO07uHrJa++w50bmufOs4+znL+a7L2Hx7Cb6wAzyKoDwJBHTi/9o5OvfD6HjkP1+q+72RqztSxHr2e23CrtqWu9ym8q6lKn5z43KPXNJOBbhh0zs2D5ZybgLND/rk3FWffEPTeKPQ77mtuHjTwDcv5j7NecE39t53vuK+pacA6z39j9/U1dd+8df3Qu262LBdfU5/jZOn+9+t7HuvZ27uXhfX5+Q5wzYHO3VNrn3P3+feSSmr2652jW9Rx1ntN+I0VMLRG/HhGAJ4RFBign9w6Vc9v2q3//XeJAqwWObISPXa9rokysYoLjVVW91z4vdUlqmut77dvRHC4Vky7T4Zh9K6FdxuG1P1Zz3ZjgG06++uSjO6/8Nqzv9HzWffHfc7Tb9v5jvvqmv1qkiENdJ4Bj/ummiS33JKhs8791cdduxhyn33N89TUs2/fc/et9auP3eec+9yaevY969xGxwA1nV1397Zzfhbq83PrX9O5/w4Dn7v7uO7r917pnOuce/2RqN3drrfK3iGoA15EUAdwQYKDAvRfiy7Xc38v0rrNTgUEWDUzI2HYrn9L6g0DrlH/btp3NC784tfOAxfjfDd3OuemyuhzEzDQDdMA5zF0znH9bx4Gvv75bxLdxrk3Gv1r+uqGxdBLu/9vwO97oJtjAMOHoA7ggoUEBei/b79cv/97kV7OK5bVYtGM9PhhuXbPUz3W0MIbzl7K0vM//iQ6JGrAUB4dEuWFagD0YI16N9aoAxeuubVDz/5tl45UnNbK705V9uS4Yb0+/QUMrW0VO5mqBHjYpaxR//qhxQAwgNCQQD30vWyNt4Vp7ZtdE2EAmNesxOm6M+M2RYdEyaKuJ+mEdMD7eKLejSfqwMU709Ku1a/t0onqM/qvRVM1ZWLssFyX/gI8h/4CPIMn6gCG1ZhRQfqfO7I1Nna01ry+RyVHar1dEgAAfoOgDmBQwkK7wnpCVKief3239h2r83ZJAAD4BYI6gEGLGB2sh5fkKDZilJ77x24dKGekGwAAg0VQBzAkIscE65ElOYoKD9Hv/16kshOnvF0SAACmRlAHMGSiwkL0syU5ihgdrGf/vkuHv2zwdkkAAJgWQR3AkIoOD9EjS3I0ZlSQntm4S0crmB4BAMClIKgDGHKxkaP0syU5Cg0J0OqNhTpe2ejtkgAAMB2COgCPiIsK1SNLchQcFKCnXyvUiSrCOgAAF4OgDsBjEqJH62dLchQQYNHTG3fpy5oz3i4JAADTIKgD8ChbTFdYl2HoqdcK5apt8nZJAACYAkEdgMeNjR2jR5bkqLOzK6xX1jd7uyQAAHweQR3AsEiKD9PDd2Srrb1TT7+6U9WEdQAAvhZBHcCwGW8L18N35Ki5tVNPvVao2oYWb5cEAIDPIqgDGFYpieH6nzuydaalXU+9Wqi6063eLgkAAJ9EUAcw7CaOjdBPv5ethqY2PfVaoeobCesAAJyLoA7AK1KTIvXQ96ap/nSrnn6tUKfOtHm7JAAAfApBHYDXTE6O0qrbL1fNqRat3lio002EdQAAehDUAXhV+vho/feiy1VZ16zVG3epsbnd2yUBAOATCOoAvC5zQox+cttUfVnTpGc27tKZFsI6AAAEdQA+YcrEWD146xSdqG7Us3/bpaaWDm+XBACAVxHUAfiMy1PjtGLhVB1zNer3/9il5lbCOgBg5CKoA/Ap2ZPj9MCCLB0+eVrP/aNILW2EdQDAyERQB+BzZqQn6Ie32HXwxCm9sGm3Wts7vV0SAADDjqAOwCfNyrRp+U127Tterxc27VYbYR0AMMIQ1AH4LEdWou67MVOlR+v0hzf2qL2DsA4AGDkCvV0AAHydOVPHyu029H9vl+qJDV/oTEuH6hpaFRMRoluvSdXsrERvlwgAgEcQ1AH4vKumfUsHyuv16Z6K3m01Da165e1SSSKsAwD8EktfAJhCydG6ftvaOtx64z9lXqgGAADPI6gDMIWahtbzbj/d1DbM1QAA4HksfQFgCrERIecN6z/9w2fKmhgjR5ZNOWnxCgkOGObqAAAYegR1AKZw6zWpeuXtUrV1uHu3BQdaddOVKWpq7VSB06XdZTUKDrJq+uR45dptypoYo8AAfnEIADAngjoAU+h5w+gb/ylT7QBTXxZdm6oDx+uV73Tpi9JK5TtdCgsN0syMBOXabUpLjpTVYvHmtwAAwEWxGIZheLsIX1BT0yi3e+h+FPHx4aqqOj1k5wPwlW/qr45Ot/YcqlGB06VdB6rV1uFWbMQo5dptcthtSk4IG8ZqAXPh9QvwDKvVotjYi3v9Iah3I6gD5nEx/dXc2qFdB6r1ubNCzsN1chuGkuLHyGG3KTfTprioUA9XC5gLr1+AZxDUB4GgDpjHpfZXw5k2bS+tVIHTpYMnTkmS0pIj5bDbNDMjQeGjg4e6VMB0eP0CPIOgPggEdcA8hqK/quqbVeB0Kd/p0snqMwqwWpQ1MUa5dptyJsdpVDBv4cHIxOsX4BkE9UEgqAPmMZT9ZRiGjlc2qsDpUkGJS7UNrQoOsiqne3LMFCbHYITh9QvwjEsJ6jwyAjCiWSwWjbeFa7wtXLddm6qD5aeUX1zRu0RmzKhAzcxIkCMrkckxAIBhxRP1bjxRB8xjOPqro9OtvYdrVeB0qXB/ldo63IqJCFFupk2OrEQlx4+RhdAOP8TrF+AZPFEHgCESGGBVdlqcstPi1NLWocID1SpwuvTutuN6u+CYkuLGKNduU67dpngmxwAAPIAn6t14og6Yhzf7q6GprfcPKh0s754ckxSp3O7JMRFjmBwDc+P1C/AM3kw6CAR1wDx8pb+q65tVUOJSfrFLJ6rPyGqxyD4xWrPticqeHKfQEH5pCfPxlf4C/A1BfRAI6oB5+GJ/lVc2Kt/pUoGzQjUNrQoOtCp7cpxy7TZNnRTL5BiYhi/2F+APCOqDQFAHzMOX+8ttGDpYfkoFTpe2l1aqsbldY0YF6oqMBDnsNk0eF8XkGPg0X+4vwMwI6oNAUAfMwyz91dHpVnH35JidB6rU1u5WdHiIcu02Oew2jUsIY3IMfI5Z+gswG6a+AIAPCQywalpanKalxam1rVOFB6qU73Tp/e3H9U7BMY2NHS1HVqJy7TYlMDkGAHAOrz5Rr6ys1IYNG1RUVKS9e/eqqalJGzZsUG5u7kWdp7OzUwsXLtT+/fv16KOP6p577rnoWniiDpiH2fvrdFObvthXpfziCh3onhyT+q0I5dptmpVpY3IMvMrs/QX4KtM9UT98+LDWrVunlJQUpaenq7Cw8JLOs3HjRpWXlw9xdQDgGeGjg3VdTpKuy0lS9almbSupVH6xS69+cEAbPzwo+4Ro5dptmn5ZPJNjAGAE8+orQFZWlvLz8xUdHa0PPvhAK1euvOhz1NfX64UXXtD999+vNWvWeKBKAPCcuMhQ3ehI0Y2OFJVXNarA2TXu8c//r0Qb3t2n7LQ4Oew2TZkUq6BAJscAwEji1aAeFnZxj/8H8vzzzys5OVkLFiwgqAMwteT4MCVfE6Zbr56kgydOKd/p0vaSSm0vrdSYUYGakd41Oeay8UyOAYCRwNS/U923b5/+9re/acOGDUxOAOA3LBaLJidHaXJylJbMmyznkTrlOytU4HTp46KTig4P0azMBDnsiRpvY3IMAPgrUwf1J554QvPnz9cVV1zBGnUAfikwwKrLU2N1eWqsWts6tetgtQqcLn3wRbne3XZcY2NH9457TIge7e1yAQBDyLRB/Z133lFhYaHefvvtITnfxb4L90LEx4cP+TkBdBmp/ZWcFKWbrklTw5k2fbb7pP6zs1xvfnJYb35yWOnjo3X19CRdNS1J0RGjvF0qTGyk9hfga0wZ1FtbW/XUU0/p7rvv1rhx44bknIxnBMyD/upyRVqsrkiLVc2pFm0rcSnf6dK6N/dqfd5e2SfEyMHkGFwC+gvwDNONZ7xUr776qurq6nTLLbf0LnmpqKiQJJ06dUrl5eWy2WwKCgryZpkAMCxiI0fp244UfduRohNVjcp3ulTg/GpyzLTUWDmyEjWVyTEAYCqmDOonT55UU1OTFixY0O9ra9eu1dq1a/Xvf/9bqampXqgOALwnKT5Mt3VPjik72aD84gptK6nUF/uqFBoSqCvS4+Ww25Q+PlpWK29CBQBfZoqgfuzYMUnS+PHjJUmLFi3q99dLa2pq9Ktf/Uq33Xab5s6dq8TExGGvEwB8hcViUVpSpNKSInXHvMkqOVrXG9o/2f2losKCNSvTJkeWTSm2cCbHAIAP8npQX7t2rSSprKxMkpSXl6cdO3YoIiJCS5culSTdc889kqQtW7ZIktLT05Went7nPD1LYC677DLNnz9/OEoHAFMIDLBq6qRYTZ0Uq9b2ThUdrFZ+sUsf7ijXe9uPKzFmtBx2m3LtNtlimBwDAL7C60H9+eef7/P566+/LklKSkrqDeoAgKEREhSgWZk2zcq0qbG5XV/sq1RBsUtvfnpYb356WBPHhsthT9SszARFhoV4u1wAGNEshmEM3agTE2PqC2Ae9NfQq21oUUGJSwXFLh2rbJTFItlTopVrT9T0y+I1epTXn+tgmNBfgGdcytQXgno3gjpgHvSXZ52oPqMCZ4Xyi12qPtWiwACrpqXFymFP1OWpMQoKDPB2ifAg+gvwDIL6IBDUAfOgv4aHYRg6dLJB+U6Xtpe41NDUrtCQQM3onhyTweQYv0R/AZ5BUB8EgjpgHvTX8Ot0u1VypE75Tpd27K9Sa1unIsOClZvZ9SbUCYlMjvEX9BfgGQT1QSCoA+ZBf3lXz+SYAqdLu8tq1Ok2ZOueHONgcozp0V+AZxDUB4GgDpgH/eU7zrS0a8e+KuUXV2jfsXoZkiYkhsuR1TU5JorJMaZDfwGeQVAfBII6YB70l2+qbWjRtpJK5TsrdMzVNTkmY3y0HHabZqQnMDnGJOgvwDMI6oNAUAfMg/7yfSerz6jA6VKB06XK+uauyTGpscq12zQtLZbJMT6M/gI8g6A+CAR1wDzoL/MwDEOHvmxQQbFL23onxwRoxmUJys2yKZPJMT6H/gI8g6A+CAR1wDzoL3PqdLtVerRe+cUV2rG/Si1tnYocE6xZmTY5spgc4yvoL8AzCOqDQFAHzIP+Mr+29k7tLqvR58UV2nOoRh2dhhKiQ+Wwd417HBs7xtsljlj0F+AZBPVBIKgD5kF/+ZeeyTEFTpdKj9bJkJSSGC6H3aZZmTZFhzM5ZjjRX4BnENQHgaAOmAf95b/qTrdqW4lL+U6XjlaclkVSRkq0cu02XZEer9Gjgrxdot+jvwDPIKgPAkEdMA/6a2T4sqZrcky+06XKumYFBlh0eWqcHHabLk+NVXAQk2M8gf4CPIOgPggEdcA86K+RxTAMHak4rc+LK7StpFINZ9o0KjhAMy6LlyMrURkpUQqwWr1dpt+gvwDPuJSgzl+fAAD4NIvFooljIzRxbITumDtZJcfqVFDs0o79lfpsb4UixgRrVkbXuMdJYyOYHAPAb/BEvRtP1AHzoL8gfTU5psDpUlFZddfkmKhQ5dq7xj0yOebS0F+AZ7D0ZRAI6oB50F84V1P35Jj8sybHjLeFyWFPVK6dyTEXg/4CPIOgPggEdcA86C98nbrTrdpeWqn84god6Z4ckz4+So6sRM1Ij9cYJsd8LfoL8AyC+iAQ1AHzoL9woSpqm7omxxRXyFXXrACrRZenxirXblN2WhyTYwZAfwGeQVAfBII6YB70Fy5Wz+SYAqdLBSUunWpsU0jP5Bi7TZkTopkc043+AjyDqS8AAAzg7Mkx37suTaXH6pTvdGnHvipt3VuhiNFBmplpk8Nu06RvMTkGgG/giXo3nqgD5kF/Yai0d3Rqd1mt8p0VKjpYo45Ot+KjRinXniiH3aZvxY28yTH0F+AZLH0ZBII6YB70FzyhqaVDO/dXKd9ZoZKjdTIMaXxCmHKzbMrNtCkmYpS3SxwW9BfgGQT1QSCoA+ZBf8HT6htbtb2kUvlOlw5/2SCLpMvGRSk3y6Yr0hMUFuq/k2PoL8AzCOqDQFAHzIP+wnBydU+O+dzpkqu2SQFWi6ZOipUjy6ZpaXEK8bPJMfQX4BkE9UEgqAPmQX/BGwzD0DFXoz4vrtC2EpfquyfHTJ8cL0eWTXY/mRxDfwGewdQXAAA8xGKxKCUxXCmJ4fredWnad7xeBc4KfVFapc+LKxQ+OkgzMxLkyEpUKpNjAAwBnqh344k6YB70F3xJe4dbew7VKN/pUtHBarV3uBUXOUq59q5xj0nxF/cEzdvoL8AzWPoyCAR1wDzoL/iq5taeyTEuOY/UyjCkcQlhcthtmpVpU2yk70+Oob8AzyCoDwJBHTAP+gtmcOpMm7aVuFTgdOnQyQZJ0mXJkXJkJeqKDN+dHEN/AZ5BUB8EgjpgHvQXzKayrmtyTL7TpS9ruibHTJkYI0dWorLT4hQS7DuTY+gvwDMI6oNAUAfMg/6CWfVMjilwulRQ4lLd6VaFBAUo57I4Oew22SfEKDDAu5Nj6C/AM5j6AgCADzt7csyia1O1/3i98p0ufVFaqfxil8JCgzQzM0EOu02pSZGyMjkGGNF4ot6NJ+qAedBf8DftHW7t7Z4cs6t7ckxsxCg5smzKtduUPIyTY+gvwDNY+jIIBHXAPOgv+LOeyTEFTpeKuyfHJMePkSMrUbMyExQXGerR69NfgGcQ1AeBoA6YB/2FkeLUmTZt754cU9Y9OWZycqQcdpuuyEhQ+OjgIb8m/QV4BkF9EAjqgHnQXxiJKuubuybHFFf0To7Jmhgjh92mnMnxQzY5hv4CPIM3kwIA4KcSokJ185UTdNPsFB2vbFS+s+tJ++6yGgUHWTV9crxy7TZlTfT+5BgAQ4OgDgCAiVgsFo23hWu8rWtyzIGzJ8c4uyfHZCQo125TWjKTYwAzY+lLN5a+AOZBfwH9dXS6tedQjQqcLu06UK22DrdiI0I0y27TbHuikhMu7Ffu9BfgGSx9AQBghAoMsCpncrxyJserubVDuw5U63Nnhd4tOK63848pKX6MHHabcjNtiovy7OQYAEODJ+rdeKIOmAf9BVy4hjNt2l5aqQKnSwdPnJIkpZ01OSaie3LM58UVeuM/ZaptaFVMRIhuvSZVs7MSvVk64FeY+jIIBHXAPOgv4NJU9UyOcbp0svqMrJauyTFxkSH6bE+F2jrcvfsGB1r1/W9nENaBIUJQHwSCOmAe9BcwOIZh6HhlowqcLhWUuFTb0DrgfrERIXp6xZxhrg7wT5cS1JnfBADACNMzOeb269L01I+vPO9+NecJ8ACGB0EdAIARzGqxKDYiZMCvnW87gOFBUAcAYIS79ZpUBQf2jQTBgVbdek2qlyoCIDGeEQCAEa/nDaNMfQF8C0EdAABodlaiZmcl8mZtwIew9AUAAADwQQR1AAAAwAcR1AEAAAAfRFAHAAAAfBBBHQAAAPBBBHUAAADABxHUAQAAAB9EUAcAAAB8EEEdAAAA8EH8ZdJuVqvFFOcE0IX+AjyH/gKG3qX0lcUwDMMDtQAAAAAYBJa+AAAAAD6IoA4AAAD4III6AAAA4IMI6gAAAIAPIqgDAAAAPoigDgAAAPgggjoAAADggwjqAAAAgA8iqAMAAAA+iKAOAAAA+KBAbxfgTyorK7VhwwYVFRVp7969ampq0oYNG5Sbm+vt0gBT2717t/75z3+qoKBAJ0+eVFRUlHJycrRq1SqlpKR4uzzA1Pbs2aM//vGPcjqdqqmpUXh4uDIyMrRy5UpNnz7d2+UBfmfdunVavXq1MjIylJeX97X7EtSH0OHDh7Vu3TqlpKQoPT1dhYWF3i4J8Avr16/Xzp07dcMNNyg9PV1VVVX661//qoULF2rTpk1KTU31domAaR0/flydnZ26/fbbFR8fr9OnT+tf//qXli5dqnXr1mnOnDneLhHwG1VVVXrppZc0evToC9rfYhiG4eGaRozGxka1t7crOjpaH3zwgVauXMkTdWAI7Ny5U1OmTFFwcHDvtiNHjujmm2/Wd77zHT355JNerA7wP83NzZo/f76mTJmil19+2dvlAH7j5z//uU6ePCnDMNTQ0PCNT9RZoz6EwsLCFB0d7e0yAL8zffr0PiFdkiZMmKDJkyerrKzMS1UB/is0NFQxMTFqaGjwdimA39i9e7feeustPfrooxd8DEEdgCkZhqHq6mpujoEh0tjYqNraWh06dEjPPvus9u/fr9mzZ3u7LMAvGIahxx9/XAsXLlRmZuYFH8cadQCm9NZbb8nlcumhhx7ydimAX3jsscf07rvvSpKCgoJ0xx136IEHHvByVYB/ePPNN3Xw4EG9+OKLF3UcQR2A6ZSVlek3v/mNZsyYoQULFni7HMAvrFy5UosXL1ZFRYXy8vLU1tam9vb2fsvOAFycxsZGPfPMM/rhD3+ohISEizqWpS8ATKWqqko/+tGPFBkZqeeff15WK/83BgyF9PR0zZkzR7fddpv+/Oc/q7i4+KLW0gIY2EsvvaSgoCDde++9F30sr3AATOP06dNavny5Tp8+rfXr1ys+Pt7bJQF+KSgoSPPmzdN7772nlpYWb5cDmFZlZaVeeeUV3XnnnaqurlZ5ebnKy8vV2tqq9vZ2lZeX69SpU+c9nqUvAEyhtbVVDzzwgI4cOaK//OUvmjRpkrdLAvxaS0uLDMPQmTNnNGrUKG+XA5hSTU2N2tvbtXr1aq1evbrf1+fNm6fly5fr4YcfHvB4gjoAn9fZ2alVq1Zp165dWrt2rbKzs71dEuA3amtrFRMT02dbY2Oj3n33XY0dO1axsbFeqgwwv+Tk5AHfQPrcc8+pqalJjz32mCZMmHDe4wnqQ2zt2rWS1DvbOS8vTzt27FBERISWLl3qzdIA03ryySe1ZcsWXXfddaqvr+/zByLGjBmj+fPne7E6wNxWrVqlkJAQ5eTkKD4+Xl9++aXeeOMNVVRU6Nlnn/V2eYCphYeHD/ga9corryggIOAbX7/4y6RDLD09fcDtSUlJ2rJlyzBXA/iHZcuWadu2bQN+jd4CBmfTpk3Ky8vTwYMH1dDQoPDwcGVnZ+u+++7TrFmzvF0e4JeWLVt2QX+ZlKAOAAAA+CCmvgAAAAA+iKAOAAAA+CCCOgAAAOCDCOoAAACADyKoAwAAAD6IoA4AAAD4III6AAAA4IMI6gAAr1m2bJnmzp3r7TIAwCcFersAAMDQKigo0N13333erwcEBMjpdA5jRQCAS0FQBwA/ddNNN+nqq6/ut91q5ZepAGAGBHUA8FN2u10LFizwdhkAgEvEYxUAGKHKy8uVnp6uNWvWaPPmzbr55ps1depUXXvttVqzZo06Ojr6HVNaWqqVK1cqNzdXU6dO1Y033qh169aps7Oz375VVVV64oknNG/ePE2ZMkWzZ8/Wvffeq88++6zfvi6XSz/96U81c+ZMTZs2Tffff78OHz7ske8bAMyCJ+oA4Keam5tVW1vbb3twcLDCwsJ6P9+yZYuOHz+uu+66S3FxcdqyZYv+8Ic/6OTJk/rd737Xu9+ePXu0bNkyBQYG9u770UcfafXq1SotLdUzzzzTu295ebmWLFmimpoaLViwQFOr7/eIAAADf0lEQVSmTFFzc7OKioq0detWzZkzp3ffpqYmLV26VNOmTdNDDz2k8vJybdiwQStWrNDmzZsVEBDgoZ8QAPg2gjoA+Kk1a9ZozZo1/bZfe+21evnll3s/Ly0t1aZNm5SVlSVJWrp0qR588EG98cYbWrx4sbKzsyVJv/3tb9XW1qaNGzcqIyOjd99Vq1Zp8+bNWrRokWbPni1J+vWvf63KykqtX79eV111VZ/ru93uPp/X1dXp/vvv1/Lly3u3xcTE6Omnn9bWrVv7HQ8AIwVBHQD81OLFi3XDDTf02x4TE9Pn8yuvvLI3pEuSxWLRD37wA33wwQd6//33lZ2drZqaGhUWFur666/vDek9+/74xz/WO++8o/fff1+zZ89WfX29PvnkE1111VUDhuxz38xqtVr7TalxOBySpKNHjxLUAYxYBHUA8FMpKSm68sorv3G/1NTUftvS0tIkScePH5fUtZTl7O1nmzRpkqxWa+++x44dk2EYstvtF1RnQkKCQkJC+myLioqSJNXX11/QOQDAH/FmUgCAV33dGnTDMIaxEgDwLQR1ABjhysrK+m07ePCgJGncuHGSpOTk5D7bz3bo0CG53e7efcePHy+LxaKSkhJPlQwAIwJBHQBGuK1bt6q4uLj3c8MwtH79eknS/PnzJUmxsbHKycnRRx99pP379/fZ909/+pMk6frrr5fUtWzl6quv1scff6ytW7f2ux5PyQHgwrBGHQD8lNPpVF5e3oBf6wngkpSRkaHvf//7uuuuuxQfH68PP/xQW7du1YIFC5STk9O73y9+8QstW7ZMd911l+68807Fx8fro48+0qeffqqbbrqpd+KLJP3yl7+U0+nU8uXLtXDhQmVlZam1tVVFRUVKSkrSI4884rlvHAD8BEEdAPzU5s2btXnz5gG/9t577/WuDZ87d64mTpyol19+WYcPH1ZsbKxWrFihFStW9Dlm6tSp2rhxo1544QW99tprampq0rhx4/Twww/rvvvu67PvuHHj9Prrr+vFF1/Uxx9/rLy8PEVERCgjI0OLFy/2zDcMAH7GYvA7SAAYkcrLyzVv3jw9+OCD+slPfuLtcgAA52CNOgAAAOCDCOoAAACADyKoAwAAAD6INeoAAACAD+KJOgAAAOCDCOoAAACADyKoAwAAAD6IoA4AAAD4III6AAAA4IMI6gAAAIAP+v+bZum6YD+rBgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Train Loss</th>\n",
              "      <th>Valid Loss</th>\n",
              "      <th>Valid Acc</th>\n",
              "      <th>Train Time</th>\n",
              "      <th>Valid Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.7613</td>\n",
              "      <td>1.5350</td>\n",
              "      <td>0.4400</td>\n",
              "      <td>0:02:08</td>\n",
              "      <td>0:00:13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.4353</td>\n",
              "      <td>1.4869</td>\n",
              "      <td>0.4659</td>\n",
              "      <td>0:02:07</td>\n",
              "      <td>0:00:13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.3226</td>\n",
              "      <td>1.4852</td>\n",
              "      <td>0.4644</td>\n",
              "      <td>0:02:07</td>\n",
              "      <td>0:00:13</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Train Loss  Valid Loss  Valid Acc Train Time Valid Time\n",
              "epoch                                                         \n",
              "1          1.7613      1.5350     0.4400    0:02:08    0:00:13\n",
              "2          1.4353      1.4869     0.4659    0:02:07    0:00:13\n",
              "3          1.3226      1.4852     0.4644    0:02:07    0:00:13"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DrTi4opkZOFU",
        "colab_type": "code",
        "outputId": "376f99fa-9f49-4550-cc0e-35eb5c89336e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        }
      },
      "source": [
        "predictions, true_labels = predict(model, prediction_dataloader)\n",
        "f1, acc = evaluate(predictions, true_labels, avg='macro')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "Accuracy:        0.4733\n",
            "macro Precision: 0.4637\n",
            "macro Recall:    0.4637\n",
            "macro F1 score:  0.4637 \n",
            "\n",
            "[[118  26  62   4  14  46  12   7]\n",
            " [ 16 221  35  33  36   7  28  42]\n",
            " [ 40  35 187   5  27  30   8  34]\n",
            " [  6  93   5  66  15   6  34  19]\n",
            " [ 19  31  19   8 107   4  45  25]\n",
            " [ 39   7  48   3   4 180  20  13]\n",
            " [ 11  32  11  10  29   5 145  20]\n",
            " [ 10  35  26   4  24  20  25 119]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q4musfsEZbDb",
        "colab_type": "text"
      },
      "source": [
        "## Cross-Validation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cxmr4EBqZCms",
        "colab_type": "code",
        "outputId": "ed22f00a-beb2-49b6-b36a-ad6d16f919da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 995
        }
      },
      "source": [
        "# if 5 folds:  20% for testing (1 fold), 70.00% for training and 10.00% for validation\n",
        "# if 10 folds: 10% for testing (1 fold), 78.75% for training and 11.25% for validation\n",
        "\n",
        "if CROSS_VALIDATION:\n",
        "  kf = KFold(n_splits=NUM_FOLDS, random_state=SEED, shuffle=True)\n",
        "\n",
        "  mf1s = []\n",
        "  accs = []\n",
        "  fold_num = 1\n",
        "  for train_index, test_index in kf.split(df_data):\n",
        "      print(\"##### Fold number:\", fold_num, \"#####\")\n",
        "      fold_num += 1\n",
        "      train_df = df_data.iloc[train_index]\n",
        "      test_df  = df_data.iloc[test_index]\n",
        "\n",
        "      train_df, dev_df = train_test_split(train_df, test_size=0.125) # change percentage (hyperparams?)\n",
        "\n",
        "      X_train, y_train = train_df.sentence.values, train_df.label.values\n",
        "      X_dev, y_dev = dev_df.sentence.values, dev_df.label.values\n",
        "      X_test, y_test = test_df.sentence.values, test_df.label.values\n",
        "      \n",
        "      train_dataloader      = prepare_data(X_train, y_train, 1)\n",
        "      dev_dataloader        = prepare_data(X_dev, y_dev, 0)\n",
        "      prediction_dataloader = prepare_data(X_test, y_test, 0)\n",
        "\n",
        "      training_stats = train(model, train_dataloader, dev_dataloader, epochs=EPOCHS, verbose=False)\n",
        "      predictions, true_labels = predict(model, prediction_dataloader)\n",
        "      mf1, acc = evaluate(predictions, true_labels, verbose=False)\n",
        "      mf1s.append(mf1)\n",
        "      accs.append(acc)\n",
        "  \n",
        "  print(\"#####################################################################\")\n",
        "  print(\"PARAMS: epochs:\",EPOCHS,\", lr_rate:\",LEARN_RATE,\"epsilon:\",EPSILON,\"...\")\n",
        "  print(\"#####################################################################\")\n",
        "  print(\"F1(CV):\",mf1s)\n",
        "  print(f\"Mean-folds-F1: {sum(mf1s)/len(mf1s)}\")\n",
        "  print(\"Acc(CV):\",accs)\n",
        "  print(f\"Mean-folds-Acc: {sum(accs)/len(accs)}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "##### Fold number: 1 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:40 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "##### Fold number: 2 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:40 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "##### Fold number: 3 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:40 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "##### Fold number: 4 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:40 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "##### Fold number: 5 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:40 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "#######################################\n",
            "F1(CV): [0.5248005242009097, 0.5334083621835556, 0.5326941295270323, 0.5270420834274705, 0.5330124824251088]\n",
            "Mean-folds-F1: 0.5301915163528154\n",
            "Acc(CV): [0.5356921166977033, 0.542933995447962, 0.5383819573763707, 0.5356921166977033, 0.5387003311258278]\n",
            "Mean-folds-Acc: 0.5382801034691134\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}