Levels_train_inference.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/calisolo/Levels_image_captioning_NICE/blob/master/Levels_train_inference.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "APlAuSyWvHhh",
        "outputId": "705eecb9-8ed3-44c1-8a90-72c6679d6d64"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n",
            "Cloning into 'Levels_image_captioning_NICE'...\n",
            "remote: Enumerating objects: 698, done.\u001b[K\n",
            "remote: Counting objects: 100% (168/168), done.\u001b[K\n",
            "remote: Compressing objects: 100% (121/121), done.\u001b[K\n",
            "remote: Total 698 (delta 80), reused 121 (delta 46), pack-reused 530\u001b[K\n",
            "Receiving objects: 100% (698/698), 89.39 MiB | 16.52 MiB/s, done.\n",
            "Resolving deltas: 100% (311/311), done.\n",
            "Cloning into 'NICE_tsv'...\n",
            "remote: Enumerating objects: 9, done.\u001b[K\n",
            "remote: Counting objects: 100% (9/9), done.\u001b[K\n",
            "remote: Compressing objects: 100% (8/8), done.\u001b[K\n",
            "remote: Total 9 (delta 1), reused 0 (delta 0), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (9/9), 1.25 KiB | 642.00 KiB/s, done.\n",
            "Filtering content: 100% (2/2), 890.00 MiB | 20.03 MiB/s, done.\n",
            "/content/Levels_image_captioning_NICE\n"
          ]
        }
      ],
      "source": [
        "%cd /content\n",
        "\n",
        "!git clone https://github.com/calisolo/Levels_image_captioning_NICE.git\n",
        "!git clone https://huggingface.co/datasets/calisolo/NICE_tsv/  #dataset\n",
        "\n",
        "%cd ./Levels_image_captioning_NICE"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ilOUol9X2OTP",
        "outputId": "3908ee3b-98f2-4337-c111-f1abd4ed06dc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers==4.20.0\n",
            "  Downloading transformers-4.20.0-py3-none-any.whl (4.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.4/4.4 MB\u001b[0m \u001b[31m84.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0 (from transformers==4.20.0)\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m28.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1 (from transformers==4.20.0)\n",
            "  Downloading tokenizers-0.12.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.6/6.6 MB\u001b[0m \u001b[31m112.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.20.0) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.20.0) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers==4.20.0) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.20.0) (3.4)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.14.1 tokenizers-0.12.1 transformers-4.20.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting loguru\n",
            "  Downloading loguru-0.7.0-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: loguru\n",
            "Successfully installed loguru-0.7.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pycocoevalcap\n",
            "  Downloading pycocoevalcap-1.2-py3-none-any.whl (104.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.3/104.3 MB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pycocotools>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from pycocoevalcap) (2.0.6)\n",
            "Requirement already satisfied: matplotlib>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from pycocotools>=2.0.2->pycocoevalcap) (3.7.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from pycocotools>=2.0.2->pycocoevalcap) (1.22.4)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (1.0.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (4.39.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (1.4.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (23.1)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (8.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (3.0.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib>=2.1.0->pycocotools>=2.0.2->pycocoevalcap) (1.16.0)\n",
            "Installing collected packages: pycocoevalcap\n",
            "Successfully installed pycocoevalcap-1.2\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers==4.20.0\n",
        "!pip install loguru\n",
        "!pip install pycocoevalcap"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "vgoCvmiIvqQ0"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import torch\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-1CoywWiU8em"
      },
      "source": [
        "##make tokenizer!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vaacjeoXMiHi"
      },
      "outputs": [],
      "source": [
        "#카이밍초기화 필요? torch.nn.init.kaiming_uniform_(model.score.weight)\n",
        "\n",
        "intent_list = []\n",
        "meaningful_tags = ['[cosHint lv4]','[cosHint lv3]','[cosHint lv2]','[cosHint lv1]', '[diffHint lv3]','[diffHint lv2]','[diffHint lv1]','[shot_style]','[Location]','[NULL]']\n",
        "cat_tags = ['[outdoors]', '[misc]', '[office]', '[electronics]', '[food]', '[indoors]','[medical]', '[stocky_setting]', '[animals]']\n",
        "#intent_list += real_add_word\n",
        "intent_list += meaningful_tags\n",
        "intent_list += cat_tags"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vpaffh5CNg0Q",
        "outputId": "7f9b9399-e3d4-4dcd-873e-84f5912be7b4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[cosHint lv4]',\n",
              " '[cosHint lv3]',\n",
              " '[cosHint lv2]',\n",
              " '[cosHint lv1]',\n",
              " '[diffHint lv3]',\n",
              " '[diffHint lv2]',\n",
              " '[diffHint lv1]',\n",
              " '[shot_style]',\n",
              " '[Location]',\n",
              " '[NULL]',\n",
              " '[outdoors]',\n",
              " '[misc]',\n",
              " '[office]',\n",
              " '[electronics]',\n",
              " '[food]',\n",
              " '[indoors]',\n",
              " '[medical]',\n",
              " '[stocky_setting]',\n",
              " '[animals]']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "intent_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 260,
          "referenced_widgets": [
            "ba59384858e448c1a77dd7046329e936",
            "114b79bfbcc04fbe88f022d3d6a16950",
            "ddfb571f11e04eaa88762379f53a17ab",
            "678b7fb362ad4ab88943aeafcc1ac9aa",
            "576c4643c5584d8cab69b2fc44993d75",
            "9cd3bc45c234493eaac60627ac333903",
            "b149199c650d4b709a68c8a136876b41",
            "cd001b788ada4bb79b0e41b53590f020",
            "ccb6c43e7edb4308808839d66c930286",
            "ae0ecfca6d48481788076a9ee1a1a8c1",
            "42959e14402048b6ad4779379566235a",
            "cffdd4e4e7974cb6b89a55a6f020e0bc",
            "079c3986fb77465eb3ae25e29d1ac74b",
            "bdf99afe326347b88b7e243a56e9c8c3",
            "9efc6fc7a9504e38895d02237cfa9865",
            "fa559e74e3dd45c7b99e8df890c0b1f8",
            "f67ae791b94a4080ae64d929bfd540e2",
            "34bdb6732cbd4aa29a6a1f09d698127e",
            "b9ec3753f1564919b55a870c7f214848",
            "7e543fdc5e8d4961ae779e6bcae81b67",
            "12a894a7eb8c49bca26abe13ba73cd94",
            "8e6bf629eec745d28cb1b4a571cb8018",
            "928f936b952b4a2fb616c430efa3348c",
            "0a8bdc7012b94e3dbd70c277bf665c4e",
            "654cf185b7944a8da5c675d958f55bff",
            "1f447c78b42146e788a044dfacca0e32",
            "788cbf7ba1e640bbbcc06cf2f3b594f8",
            "29e50b0d2b174bfb8947a2d7d63d3090",
            "fe3a0da63960400181974dddb320e552",
            "b88260c780434d5a9916dc269350a61a",
            "80f93cb699984e92b0101c6bda2ba41b",
            "0f032ee70e534ca4b9caff416c8bc479",
            "d43ecd3bc74d4756b63fe7df079c6db7"
          ]
        },
        "id": "uZianB33SK60",
        "outputId": "4eff1593-8cb1-43d7-ee9d-6e4a59aa4ded"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "OFA-Sys/ofa-huge\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/878k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ba59384858e448c1a77dd7046329e936"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cffdd4e4e7974cb6b89a55a6f020e0bc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.36k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "928f936b952b4a2fb616c430efa3348c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('./vocab/tokenizer_config.json',\n",
              " './vocab/special_tokens_map.json',\n",
              " './vocab/vocab.json',\n",
              " './vocab/merges.txt',\n",
              " './vocab/added_tokens.json')"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "from component.ofa.tokenization_ofa import OFATokenizer\n",
        "tokenizer = OFATokenizer.from_pretrained('OFA-Sys/ofa-huge' ,additional_special_tokens = intent_list)\n",
        "tokenizer.save_pretrained('./vocab')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QQboTqVC8ww1"
      },
      "source": [
        "## Train"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "py4ChRO5wONY",
        "outputId": "5101763c-5f0b-446f-f823-5cf104419579"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-05-27 07:27:09.184200: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
            "2023-05-27 07:27:09.239571: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2023-05-27 07:27:10.204973: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "Downloading: 100% 1.36k/1.36k [00:00<00:00, 1.25MB/s]\n",
            "Downloading: 100% 4.09G/4.09G [11:16<00:00, 6.50MB/s]\n",
            "/usr/local/lib/python3.10/dist-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3483.)\n",
            "  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n",
            "\u001b[32m2023-05-27 07:38:45.785\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mmain\u001b[0m:\u001b[36m52\u001b[0m - \u001b[1mTotal training params: 929.49M\u001b[0m\n",
            "\u001b[32m2023-05-27 07:38:45.785\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mcomponent.dataset\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m15\u001b[0m - \u001b[1mloading data from:./data/train_caption_catAdded_2.jsonl and ../NICE_tsv/train_image.tsv\u001b[0m\n",
            "100% 5000/5000 [00:00<00:00, 32003.32it/s]\n",
            "100% 5000/5000 [00:00<00:00, 25890.64it/s]\n",
            "\u001b[32m2023-05-27 07:38:46.277\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mcomponent.dataset\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m37\u001b[0m - \u001b[1mlen of data:5000\u001b[0m\n",
            "***** Running training *****\n",
            "  Num examples = 5000\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 12\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 12\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 8340\n",
            "  0% 8/8340 [00:19<3:43:07,  1.61s/it]Exception ignored in: <function _MultiProcessingDataLoaderIter.__del__ at 0x7f5d694f83a0>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\", line 1478, in __del__\n",
            "    self._shutdown_workers()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\", line 1423, in _shutdown_workers\n",
            "    self._worker_result_queue.put((None, None))\n",
            "  File \"/usr/lib/python3.10/multiprocessing/queues.py\", line 94, in put\n",
            "    self._start_thread()\n",
            "  File \"/usr/lib/python3.10/multiprocessing/queues.py\", line 166, in _start_thread\n",
            "    self._thread = threading.Thread(\n",
            "  File \"/usr/lib/python3.10/threading.py\", line 881, in __init__\n",
            "    self._invoke_excepthook = _make_invoke_excepthook()\n",
            "KeyboardInterrupt: \n",
            "Traceback (most recent call last):\n",
            "  File \"/content/Levels_image_captioning_NICE/train.py\", line 92, in <module>\n",
            "    main()\n",
            "  File \"/content/Levels_image_captioning_NICE/train.py\", line 75, in main\n",
            "    train_result = trainer.train()\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\", line 1409, in train\n",
            "    return inner_training_loop(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\", line 1651, in _inner_training_loop\n",
            "    tr_loss_step = self.training_step(model, inputs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\", line 2345, in training_step\n",
            "    loss = self.compute_loss(model, inputs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\", line 2377, in compute_loss\n",
            "    outputs = model(**inputs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/Levels_image_captioning_NICE/component/ofa/modeling_ofa.py\", line 2080, in forward\n",
            "    encoder_outputs = self.encoder(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/Levels_image_captioning_NICE/component/ofa/modeling_ofa.py\", line 1276, in forward\n",
            "    hidden_outputs = layer(x, attention_mask if has_pads else None, attn_bias=self_attn_bias, output_attentions=output_attentions)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/Levels_image_captioning_NICE/component/ofa/modeling_ofa.py\", line 507, in forward\n",
            "    hidden_states, attn_weights, _ = self.self_attn(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/content/Levels_image_captioning_NICE/component/ofa/modeling_ofa.py\", line 392, in forward\n",
            "    query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)\n",
            "  File \"/content/Levels_image_captioning_NICE/component/ofa/modeling_ofa.py\", line 333, in _shape\n",
            "    return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()\n",
            "KeyboardInterrupt\n",
            "  0% 8/8340 [00:20<5:59:42,  2.59s/it]\n",
            "^C\n"
          ]
        }
      ],
      "source": [
        "#Train\n",
        "#freeze encoder 검증\n",
        "#freeze ->unfreeze?  unfreeze ->freeze?\n",
        "# freeze word embed, warmupstep,max_len, caption에 제일좋은 가중치\n",
        "#  scst? , 대소문자, fp 32, 길이비교( 다소 클수있음),tag?\n",
        "# cosine 어닐링 구현형태 검증\n",
        "\n",
        "#제출본에서의 역공학\n",
        "#{'testlen': 235366, 'reflen': 238872, 'guess': [235366, 213989, 192613, 171240], 'correct': [129549, 78068, 52116, 36133]} BLEU에 대한 평가내역.. 딱히 이걸로 유추할수있는 정보가있나?.. 단어생성스타일은 매우 유사해졌다는정도.. \n",
        "#BLEU/Cider metric의 관계확인 #BLEU 고득점 = 말투는 비슷한데 다른 정답에서 쓴 단어를 찾아낸경우 #추가제출할때 내 제출본과 BLEU검증가능할듯\n",
        "#ratio: 0.9853226832780695\n",
        "# 토큰 re처리 # 안해도될듯\n",
        "# WARNING: Untokenizable: � (U+FFFD, decimal: 65533)\n",
        "# PTBTokenizer tokenized 260248 tokens at 935525.16 tokens per second. #정답\n",
        "# Apr 23, 2023 10:49:44 AM edu.stanford.nlp.process.PTBLexer next\n",
        "# WARNING: Untokenizable: � (U+FFFD, decimal: 65533)\n",
        "# PTBTokenizer tokenized 256742 tokens at 930426.72 tokens per second. #4/23제출본\n",
        "\n",
        "#30epochs = 12510 steps  -> 20epochs = 8340 steps  decay  \n",
        "\n",
        "\n",
        "#import re\n",
        "#result = re.sub(r'[^\\w\\s]', '', result).strip()\n",
        "#num_beams=5, no_repeat_ngram_size=3, use_cache=False \n",
        "!CUDA_VISIBLE_DEVICES=0 python train.py --train_args_file train_args/train_ofa.json "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ooehvzcltgHA"
      },
      "source": [
        "## Inference\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JgDOIKyQFlZi"
      },
      "outputs": [],
      "source": [
        "val_set = pd.read_csv(f'data/nice-val-5k.csv')\n",
        "inference_model = 'calisolo/OFA_huge_NICE_captioning'\n",
        "cosine = pd.read_csv('./data/encoder/test_fully_parsed_final_2.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KkeLzVuXGdiW"
      },
      "outputs": [],
      "source": [
        "val_dict = {}\n",
        "for index, i in enumerate(val_set['public_id']):\n",
        "  val_dict[i] = val_set['caption_gt'][index]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ONaXdNU6GnP4"
      },
      "outputs": [],
      "source": [
        "#make levels hint\n",
        "\n",
        "import ast\n",
        "caption_result = {}\n",
        "\n",
        "for index, public_id in enumerate(cosine['0']):\n",
        "  processed_data = ''\n",
        "\n",
        "\n",
        "  processed_data = cosine['cos1'][index] + cosine['diff1'][index] + cosine['cat1'][index]+ val_dict[cosine['key1'][index]]\n",
        "  processed_data += cosine['cos2'][index] + cosine['diff2'][index] + cosine['cat2'][index]+val_dict[cosine['key2'][index]]\n",
        "  processed_data += cosine['cos3'][index] + cosine['diff3'][index] + cosine['cat3'][index]+val_dict[cosine['key3'][index]]\n",
        "  processed_data += cosine['cos4'][index] + cosine['diff4'][index] + cosine['cat4'][index]+val_dict[cosine['key4'][index]]\n",
        "  processed_data += '[shot_style]'\n",
        "\n",
        "  \n",
        "  shotlist = ast.literal_eval(cosine['shots'][index])\n",
        "  locationlist = ast.literal_eval(cosine['locations'][index])\n",
        "  if shotlist == []:\n",
        "    processed_data += '[NULL]'\n",
        "  else:\n",
        "    for shot in shotlist:\n",
        "      processed_data += shot + ' | '\n",
        "  processed_data += '[Location]'\n",
        "  if locationlist == []:\n",
        "    processed_data += '[NULL]'\n",
        "  else:\n",
        "    for location in locationlist:\n",
        "      processed_data += location + ' | '\n",
        "  caption_result[str(public_id)] = processed_data\n",
        "  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nMIdUzLjRlO5",
        "outputId": "9f43ac99-bc9b-458b-b12d-73307ecaf181"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3483.)\n",
            "  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "../drive/MyDrive/NICE/model/0501_lastcoin/checkpoint-final\n",
            "./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 21377/21377 [3:24:20<00:00,  1.74it/s]\n"
          ]
        }
      ],
      "source": [
        "#inference\n",
        "from component.ofa.modeling_ofa import OFAModelForCaption, OFAModel\n",
        "from component.ofa.tokenization_ofa import OFATokenizer \n",
        "from torchvision import transforms\n",
        "from PIL import Image\n",
        "from tqdm import tqdm\n",
        "import torch\n",
        "import os\n",
        "import base64\n",
        "from datetime import date, datetime\n",
        "from io import BytesIO\n",
        "import datetime\n",
        "test = True\n",
        "\n",
        "csv_path = '../NICE_tsv/'\n",
        "\n",
        "if test:\n",
        "  image_file = f'{csv_path}test_image(need to delete backslash).tsv'  ## tsv(byte) read pictures faster than jpg \n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model = OFAModel.from_pretrained(inference_model).to(device)\n",
        "print(inference_model)\n",
        "tokenizer = OFATokenizer.from_pretrained('./vocab' )\n",
        "model.eval()\n",
        "\n",
        "\n",
        "mean, std = [0.5, 0.5, 0.5], [0.5, 0.5, 0.5]\n",
        "resolution = 256\n",
        "patch_resize_transform = transforms.Compose([\n",
        "        lambda image: image.convert(\"RGB\"),\n",
        "        transforms.Resize((resolution, resolution), interpolation=Image.BICUBIC),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize(mean=mean, std=std)\n",
        "    ])\n",
        "\n",
        "resultDF = pd.DataFrame(columns = ['public_id', 'caption'])\n",
        "\n",
        "iter = 0\n",
        "batch_size = 1\n",
        "pic_list = []\n",
        "save = 0\n",
        "with open(image_file, 'r', encoding='utf8') as f:\n",
        "  lines = f.readlines()\n",
        "  with torch.no_grad():\n",
        "    for line in tqdm(lines):\n",
        "      pic_id, image_content = line.split('\\t')\n",
        "      \n",
        "          \n",
        "      if test:\n",
        "        pic_id = pic_id[:-1]##didn't erase \\\\ at tsv\n",
        "\n",
        "        txt = \" what does the image describe?\"\n",
        "        txt+= caption_result[pic_id]\n",
        "        \n",
        "        inputs = tokenizer([txt], return_tensors=\"pt\").input_ids.to(device)\n",
        "        img = Image.open(BytesIO(base64.urlsafe_b64decode(image_content)))\n",
        "        \n",
        "      else:\n",
        "        img = Image.open(f'{csv_path}val/{pic_id}.jpg')\n",
        "\n",
        "      if iter == 0:\n",
        "        patch_img = patch_resize_transform(img).unsqueeze(0).to(device)\n",
        "        pic_list = [pic_id]\n",
        "      else:\n",
        "        imsi_img = patch_resize_transform(img).unsqueeze(0).to(device)\n",
        "        patch_img = torch.cat((patch_img, imsi_img), dim = 0)\n",
        "        pic_list.append(pic_id)\n",
        "    \n",
        "      iter+=1\n",
        "\n",
        "      if iter== batch_size:## last batch size issue\n",
        "        iter = 0\n",
        "        gen = model.generate(inputs.repeat(batch_size,1), patch_images=patch_img, \n",
        "                            num_beams=5, no_repeat_ngram_size=3, max_length = 40\n",
        "                            , use_cache=False )\n",
        "        gen_sentence = tokenizer.batch_decode(gen, skip_special_tokens=True)\n",
        "        \n",
        "        ## erase if you want to full inference\n",
        "        img.show()\n",
        "        print(txt)\n",
        "        print(gen_sentence)\n",
        "\n",
        "\n",
        "        batch_df = pd.DataFrame(columns = ['public_id', 'caption'])\n",
        "        for id,caption in zip(pic_list, gen_sentence):\n",
        "\n",
        "          imsiSeries = pd.Series({'public_id':id, 'caption': caption})\n",
        "          imsi_df = imsiSeries.to_frame().transpose()\n",
        "          batch_df= pd.concat([batch_df,imsi_df],ignore_index = True)\n",
        "        resultDF = pd.concat([resultDF,batch_df],ignore_index = True)\n",
        "\n",
        "      # if save %2000 == 0:  save while inferencing\n",
        "      #   resultDF.to_csv(f'../submission.csv', encoding = 'utf-8', index = False)\n",
        "      # save+=1\n",
        "\n",
        "resultDF.to_csv(f'../submission_final.csv', encoding = 'utf-8', index = False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "igrMnW2X9fDE",
        "outputId": "9f734872-876e-46ce-cb1b-24627ad4fe2b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Smiling young women sitting by fountain with one playing guitar\n",
            "Vertical shot of a male and female rock climber standing on a rock\n",
            "Couple on winter vacation sitting on sled on mountain top smiling at camera\n",
            "View of a woman standing on a beach looking out to sea\n",
            "Rooftops of idyllic village Bourdeilles Dordogne France\n",
            "View of young boy concentrating on writing\n",
            "Portrait of a young woman\n",
            "Little girl standing with her mother and brother at the beach on an overcast windy day\n",
            "Senior man sitting on a deck chair\n",
            "Medium shot of the Backside of a group of friends in swimwear walking along the beach holding surfboards\n",
            "Vertical shot of a businesswoman with luggage on a road in the desert\n",
            "Young man pitching a baseball\n",
            "A businesswoman sitting at a table thinking\n",
            "Low angle view of three young women walking with shopping bags\n",
            "Two young women leaning on car\n",
            "Lumber industry at Breves Channels Brazil\n",
            "A medium shot of a businessman and businesswoman smiling at camera while working together in front of a computer\n",
            "A mid adult woman holding a pumpkin\n",
            "Father and boys running in surf on beach\n",
            "Teenage girls taking cell phone photographs\n",
            "View from Luesener Alm Dolomite Alps Alps South Tyrol Italy\n",
            "Focused schoolgirls in private school uniform sharing computer in the computer lab\n",
            "A couple sitting at a cafe table\n",
            "A young woman looking out of a window\n",
            "View from Gornercrest Zermatt Valais Switzerland Europe\n",
            "Wine and cheese with pitcher on table outdoors\n",
            "Wide shot of a manager and a technician holding a laptop and talking to each other in a server room\n",
            "Amazon River near Uara Brazil\n",
            "Teenage girlfriends listening to music on MP3 player sharing headphones whilst relaxing and leaning against beach rocks\n",
            "cutout Of Middle Aged Male Executive Wearing Suit\n",
            "A vertical view of a happy school teacher with blonde hair working on a microscope while teaching her students in science laboratory\n",
            "A family of four holding boxes\n",
            "View of a colorful hot air balloon against blue sky Balloon Festival Albuquerque New Mexico USA\n",
            "Young man having beauty treatment\n",
            "A happy couple holding keys with selective focus on the keys\n",
            "Vertical shot of school student playing a flute in a music class with selective focus\n",
            "A portrait shot of a young businesswoman smiling at camera while talking to a colleague in office\n",
            "Portrait of a woman in a tropical garden\n",
            "Workers working on coating glass for use in production of solar panels\n",
            "Family on winter vacation sitting on snow smiling at camera\n",
            "A student disappointed by his exam results\n",
            "Vertical shot of a newborn baby girl and her mother sleeping on the bed\n",
            "View to Forte Falcone Portoferraio Island of Elba Province of Livorno Tuscany Italy\n",
            "A wide shot of a happy family sitting on a fence with bicycles kept on a side in a wildflower field\n",
            "View of hands turning knob on elevator\n",
            "Horizontal shot of a senior man applying glue to a model sailboat indoors\n",
            "Worker checking boxes at the production line of a distribution warehouse\n",
            "A horizontal portrait of a scientist in a clean suit cautiously working in a silicon wafer manufacturing laboratory\n",
            "Smiling couple riding motorcycle on driveway at camera\n",
            "Horizontal overhead shot of a father and son playing basketball in a swimming pool\n",
            "Twin brothers playing in sandpit\n",
            "Horizontal shot of summer bright blue sky with fluffy clouds\n",
            "Arrows squeezing British pound symbol\n",
            "A businessman in an office pointing at the camera and smiling\n",
            "A low angle shot of a happy senior man pulling his wife's hand while walking in an autumn park\n",
            "Close up of salad in serving dish\n",
            "Dirt road next to Dischma Brook view out of Schma Valley Davos Graubuenden Grisons Switzerland\n",
            "Rear view of a suitcase on a railway station platform Bavaria Germany Europe\n",
            "A portrait shot of a businessman and businesswoman talking in a modern office building\n",
            "Group of young friends on winter vacation smiling at camera\n",
            "Young man standing in a wheat field and using his laptop\n",
            "Couple hugging on walk in autumn countryside\n",
            "A front view of a red tractor in selective leaving behind the tracks of grass in stripe pattern in the background while mowing the field for sil\n",
            "Horizontal shot of a young woman serving herself with strawberries on a table at a backyard barbecue with her friends in the background\n",
            "Horizontal shot of a businessman doing a handstand on board desk in a meeting room as colleagues watch on\n",
            "Businessmen figurines climbing a stack of Euro coins\n",
            "Senior Couple Working In Beautiful Cottage Garden\n",
            "Low angle view of engineer holding solar panel beneath wind turbine\n",
            "Close up of apple blossoms outdoors\n",
            "High angle view of a young woman packing boxes\n",
            "A group of girls with backpacks looking at a map and compass over a wooden fence in a green field\n",
            "Business employees talking while sitting in a private jet\n",
            "Side view of two male surfers in wetsuits walking in surf on the beach\n"
          ]
        }
      ],
      "source": [
        "for i in resultDF['caption']:\n",
        "  print(i)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_j3EQXzfNbYX",
        "outputId": "65da414e-eb27-4f26-cdd3-4a38294f063c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Smiling young women sitting by fountain with one playing guitar\n",
            "Vertical shot of a male and female rock climber standing on a rock\n",
            "Couple on winter vacation sitting on sled on mountain top smiling at camera\n",
            "View of a woman looking out to sea\n",
            "Rooftops of idyllic village Bourdeilles Dordogne France\n",
            "View of young boy concentrating on writing\n",
            "Portrait of a young woman\n",
            "Little girl standing with her mother and brother at the beach on an overcast windy day\n",
            "Senior man sitting on a deck chair\n",
            "Medium shot of the Backside of a group of friends in swimwear walking along the beach holding surfboards\n",
            "Vertical shot of a businesswoman with luggage on a road in the desert\n",
            "Young man pitching a baseball\n",
            "A businesswoman sitting at a table thinking\n",
            "Low angle view of three young women walking with shopping bags\n",
            "Two young women leaning on car\n",
            "Lumber industry at Breves Channels Brazil\n",
            "A medium shot of a businessman and businesswoman smiling at camera while working together at a desk in front of a computer\n",
            "A mid adult woman holding a pumpkin\n",
            "Father and boys on beach\n",
            "Teenage girls taking cell phone photographs\n",
            "View from Luesener Alm Dolomite Alps Alps South Tyrol Italy\n",
            "Focused schoolgirls in private school uniform sharing computer in the computer lab\n",
            "Young couple talking at cafe table\n",
            "A young woman looking out of a window\n",
            "View from Gornercrest Zermatt Valais Switzerland Europe\n",
            "Wine and cheese with pitcher on table outdoors\n",
            "Wide shot of a manager and a technician holding a laptop and talking to each other in a server room\n",
            "Amazon River near Uara Brazil\n",
            "Teenage girl with cell phone\n",
            "cutout Of Middle Aged Male Executive Wearing Suit\n",
            "A vertical view of a happy school teacher with blonde hair working on a microscope while teaching her students in science laboratory\n",
            "A family of four holding boxes\n",
            "View of a colorful hot air balloon against blue sky Balloon Festival Albuquerque New Mexico USA\n",
            "Young man having beauty treatment\n",
            "A happy couple holding keys with selective focus on the keys\n",
            "Vertical shot of a music teacher playing a flute with a music class in the background\n",
            "A portrait shot of a young businesswoman smiling at camera while talking on a phone with her colleagues in the background\n",
            "Portrait of a woman in a tropical garden\n",
            "Technician arranging aluminum products on a conveyor belt in the factory\n",
            "Father and daughter making snowballs on their trip to a ski resort in the snowy mountains with the daughter preparing to throw a snowball\n",
            "A student disappointed by his exam results\n",
            "High angle shot of a baby girl and her mother sleeping on the bed\n",
            "View to Forte Falcone Portoferraio Island of Elba Province of Livorno Tuscany Italy\n",
            "A wide shot of a happy family sitting on a fence with bicycles on the side in a wildflower field\n",
            "View of hands turning knob on elevator\n",
            "Horizontal shot of a senior man applying glue to a model sailboat indoors\n",
            "Worker checking boxes on shelf in pharmacy\n"
          ]
        }
      ],
      "source": [
        "for i in resultDF['caption']:\n",
        "  print(i)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##metric Test"
      ],
      "metadata": {
        "id": "1jjaSayaEiVa"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uc1myzW46TTr"
      },
      "outputs": [],
      "source": [
        "csv_path = '../drive/MyDrive/NICE/'\n",
        "# val_set = pd.read_csv(f'{csv_path}/nice-val-5k.csv')\n",
        "# result_path= './drive/MyDrive/NICE/output'\n",
        "# resultDF = pd.read_csv(f'{result_path}/train_on_0406.csv')\n",
        "\n",
        "val_set = pd.read_csv(f'{csv_path}/output/test_utf-8.csv', encoding = 'utf-8')\n",
        "resultDF = pd.read_csv(f'{csv_path}/output/test_utf-8-sig.csv', encoding = 'utf-8')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CtrWvnI34MKb"
      },
      "outputs": [],
      "source": [
        "for index,i in enumerate(resultDF['caption']):\n",
        "  resultDF['caption'][index] = '\\ufeff' +resultDF['caption'][index] "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LVxKiEWH6_Lw"
      },
      "outputs": [],
      "source": [
        "for i in range(2500):# cider 메트릭 세부테스트\n",
        "  resultDF['caption'][i] = 'A medium shot of a happy family preparing food'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 565
        },
        "id": "U_z8n3w-nKci",
        "outputId": "eb58d5dd-93b4-4d99-a61b-046e9bf9d846"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'testlen': 295419, 'reflen': 295419, 'guess': [295419, 274042, 252665, 231288], 'correct': [295419, 274042, 252665, 231288]}\n",
            "ratio: 0.9999999999999967\n",
            "Bleu_1 : 99.99999999999933\n",
            "Bleu_2 : 99.99999999999932\n",
            "Bleu_3 : 99.9999999999993\n",
            "Bleu_4 : 99.99999999999929\n",
            "Rouge : 100.0\n",
            "[10. 10. 10. ... 10. 10. 10.]\n",
            "Cider : 1000.0\n"
          ]
        },
        {
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-138-e1a41f491266>\u001b[0m in \u001b[0;36m<cell line: 31>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Cider : {score*100}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m \u001b[0mscore\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMeteor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdict_gt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Meteor : {score*100}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pycocoevalcap/meteor/meteor.py\u001b[0m in \u001b[0;36mcompute_score\u001b[0;34m(self, gts, res)\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mimgIds\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m             \u001b[0;32massert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m             \u001b[0mstat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m             \u001b[0meval_line\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m' ||| {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/pycocoevalcap/meteor/meteor.py\u001b[0m in \u001b[0;36m_stat\u001b[0;34m(self, hypothesis_str, reference_list)\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeteor_p\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstdin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{}\\n'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscore_line\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeteor_p\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstdin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeteor_p\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstdout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhypothesis_str\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreference_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "\n",
        "\n",
        "from pycocoevalcap.bleu.bleu import Bleu\n",
        "from pycocoevalcap.rouge.rouge import Rouge\n",
        "from pycocoevalcap.cider.cider import Cider\n",
        "from pycocoevalcap.meteor.meteor import Meteor\n",
        "from pycocoevalcap.spice.spice import Spice\n",
        "\n",
        "\n",
        "combined_df = resultDF.merge(val_set, on=\"public_id\")\n",
        "\n",
        "dict_gt = {}\n",
        "dict_pred = {}\n",
        "\n",
        "for idx, row in combined_df.iterrows():\n",
        "  dict_gt[str(row[\"public_id\"])] = [row[\"caption_x\"]]\n",
        "  dict_pred[str(row[\"public_id\"])] = [row[\"caption_y\"]]\n",
        "  \n",
        "\n",
        "score, _ = Bleu(4).compute_score(dict_gt, dict_pred)\n",
        "idx = 1\n",
        "for s in score:\n",
        "  print(f'Bleu_{idx} : {s*100}')\n",
        "  idx += 1\n",
        "\n",
        "score, _ = Rouge().compute_score(dict_gt, dict_pred)\n",
        "print(f'Rouge : {score*100}')\n",
        "\n",
        "score, _ = Cider().compute_score(dict_gt, dict_pred)\n",
        "print(f'Cider : {score*100}')\n",
        "\n",
        "score, _ = Meteor().compute_score(dict_gt, dict_pred)\n",
        "print(f'Meteor : {score*100}')\n",
        "\n",
        "score, _ = Spice().compute_score(dict_gt, dict_pred)\n",
        "print(f'Spice : {score*100}')\n",
        "\n",
        "#base(chinese -> train with chinese prompt and inferencing with English prompt) Cider  24.26\n",
        "#base(chinese -> train with english prompt and inferencing with English prompt) Cider  37.21  PLM with chinese/tokenizer with Chinese\n",
        "#tiny english -> english Cider 34.44  ... maybe tokenizer problem exist\n",
        "#base english -> english Cider 96.46\n",
        "#huge english -> english Cider 120.06 -> 98.2555!\n",
        "#20 epoch -> 50 epoch  max_len 100 -> 150  Cider  399.54"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JkweaEiWF1Fm"
      },
      "outputs": [],
      "source": [
        "#tokenizer 맨앞글자 짤림..  tiny size에서 짤림.. 모델규격문제인듯\n",
        "##checkpoint 확인 .. 근데 best ckpt나 기본이나 별차이는없을듯,,  encoder는 영향있을듯\n",
        "\n",
        "#20개 토크나이저 컷\n",
        "##사진 id에 따른 caption 확인\n",
        "#category 활용성\n",
        "#bleu 가 낮은편인데 문장 길이에 영향이 있을듯?\n",
        "#PIL 로 사진읽어오는게 느린거같은데 원본코드의 방식 참조\n",
        "#앞부분답이 외워지는게 학습떄 shuffle이 되고 있나?\n",
        "#260248 tokens "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Ajm6O_bYoj_"
      },
      "source": [
        "##SCST (not working)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WU4rSSrQs6qJ",
        "outputId": "19fc8df5-e943-4a08-fcdf-a0511339ff4c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3483.)\n",
            "  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n"
          ]
        }
      ],
      "source": [
        "# model = OFAModel.from_pretrained(inference_model).to(device)\n",
        "# print(inference_model)\n",
        "from component.ofa.modeling_ofa import OFAModelForCaption, OFAModel\n",
        "from component.ofa.tokenization_ofa import OFATokenizer #tokenizerfast로 만들고싶다..\n",
        "from torchvision import transforms\n",
        "from PIL import Image\n",
        "from tqdm import tqdm\n",
        "import torch\n",
        "import os\n",
        "import base64\n",
        "from datetime import date, datetime\n",
        "from io import BytesIO\n",
        "import datetime\n",
        "\n",
        "#inference_model = '../drive/MyDrive/NICE/model/0422_train_encoder/checkpoint-10000'  #best\n",
        "#inference_model = '../drive/MyDrive/NICE/model/0424_hyperparameter/checkpoint-final'   #이미 train set에서는 만점수준..\n",
        "\n",
        "inference_model = \"../drive/MyDrive/NICE/model/cap_best/OFA-huge-caption\"\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "model = OFAModel.from_pretrained(inference_model).to(device)\n",
        "tokenizer = OFATokenizer.from_pretrained('./vocab' )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i6AZm3kbpdS7",
        "outputId": "4809b40b-fb1f-4a4c-bcac-6f3d8d038275"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\u001b[32m2023-04-29 03:13:15.899\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mcomponent.dataset\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m15\u001b[0m - \u001b[1mloading data from:./data/train_caption.jsonl and ../drive/MyDrive/NICE/train_image.tsv\u001b[0m\n",
            "100%|██████████| 5000/5000 [00:00<00:00, 34179.67it/s]\n",
            "100%|██████████| 5000/5000 [00:00<00:00, 185841.94it/s]\n",
            "\u001b[32m2023-04-29 03:13:16.924\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mcomponent.dataset\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m37\u001b[0m - \u001b[1mlen of data:5000\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "from component.dataset import CaptionDataset\n",
        "from component.datacollator import CaptionCollator\n",
        "\n",
        "train_caption_file = \"./data/train_caption.jsonl\"\n",
        "train_image_file = \"../drive/MyDrive/NICE/train_image.tsv\"\n",
        "max_seq_length = 150\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "train_dataset = CaptionDataset(train_caption_file, train_image_file)\n",
        "data_collator = CaptionCollator(tokenizer=tokenizer, max_seq_length= max_seq_length)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o9f8qHTlLOzB"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "    train_dataset,\n",
        "    shuffle=True,\n",
        "    batch_size=4,\n",
        "    collate_fn=data_collator,\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7DunGrY5yRxN"
      },
      "outputs": [],
      "source": [
        "gen_target = [] #생성 토큰\n",
        "gen_res = [] #디코드 문장\n",
        "gt_res = [] # 정답문장\n",
        "for batch in train_dataloader:\n",
        "\n",
        "  model.eval()\n",
        "  with torch.no_grad():\n",
        "    output = model.generate(input_ids = batch['input_ids'], patch_images = batch['patch_images'],num_beams=5, no_repeat_ngram_size=3, max_length = 40,\n",
        "                              use_cache=False  )\n",
        "    gen_sentence = tokenizer.batch_decode(output, skip_special_tokens=True)\n",
        "    gen_target.append(output)\n",
        "    gen_res.append(gen_sentence)\n",
        "    for _ in range(4):\n",
        "      output = model.generate(input_ids = batch['input_ids'], patch_images = batch['patch_images'],temperature = 2.0,\n",
        "                              do_sample =  True,max_length = 40,\n",
        "                            #num_beams=5, no_repeat_ngram_size=3, \n",
        "                              use_cache=False )\n",
        "      gen_sentence = tokenizer.batch_decode(output, skip_special_tokens=True)\n",
        "      gen_target.append(output)\n",
        "      gen_res.append(gen_sentence)\n",
        "  caption = batch['gt']\n",
        "  gt_index = batch['gt_index']\n",
        "  gt_res = caption\n",
        "  break"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LAlwRyyyjdFa"
      },
      "outputs": [],
      "source": [
        "dict_gt,dict_gen = {},{}\n",
        "for index, id in enumerate(gt_index):\n",
        "  for i in range(len(gen_res)):\n",
        "    dict_gen[str(id)+'_'+ str(i%5)] = [gen_res[i][index]]\n",
        "    dict_gt[str(id)+'_'+ str(i%5)] = [gt_res[index]]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EwVnfTIBlfTa",
        "outputId": "19c00ae8-a567-48e6-a831-aa6602a4d966"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[' edge of a bag', ' part of a sweater', ' pink shirt on a girl sitting down', ' edge of a dress']\n",
            "[' two stud o piece of a case', ' legoin of a collar of blue shirt with two blue buttons', ' rubybluejimmedaughters shirt here', ' word bucketvonippiahouffle sleeved dress']\n",
            "[' tip or part of suitcasebag', ' the coat has two colors on the sweater', ' small yellow tennis ball laying in the grass', ' littlettle elbow of a young girl']\n",
            "['igator pouch on the leather jacket jacketbag', ' elbow buttons areaide of shrug', ' little boy head twisted upside off', ' part of an ear']\n",
            "[' black leather handle of a briefcase bag leg zipper handlebreake', ' part of a knife', \" small pink shirt on girl tummi's neck\", ' yellow eggs piled with eggs in a basket']\n"
          ]
        }
      ],
      "source": [
        "for i in gen_res:\n",
        "  print(i)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z09am5srfv8r"
      },
      "outputs": [],
      "source": [
        "\n",
        "from pycocoevalcap.cider.cider import Cider\n",
        "# gen_res_ = [{'image_id':'1', 'caption':gen_res[0][0]}]\n",
        "# gt_res_ = [{'image_id':'1', 'caption':gt_res[0]}]\n",
        "\n",
        "cider,batch_cider_scores = Cider().compute_score(dict_gt, dict_gen)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qdh13EtijxOP"
      },
      "outputs": [],
      "source": [
        "CIDER_REWARD_WEIGHT =1\n",
        "batch_cider_scores\n",
        "scores = CIDER_REWARD_WEIGHT * batch_cider_scores"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lJsIS399qLnc",
        "outputId": "b0e85dd4-3cf8-4fa3-c91d-03af71556aaf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([-0.0296, -0.0348, -0.1435, -0.1435,  0.3513,  0.0000,  0.0000,  0.0000,\n",
            "         0.0000,  0.0000,  0.1344, -0.0554, -0.0554, -0.0554,  0.0318, -0.1301,\n",
            "        -0.1301,  0.1587, -0.1301,  0.2315], dtype=torch.float64)\n"
          ]
        }
      ],
      "source": [
        "batch_size = len(gt_res)\n",
        "gen_res_size = len(gen_res)*batch_size\n",
        "seq_per_img = gen_res_size // batch_size\n",
        "\n",
        "sc_ = scores.reshape(batch_size, seq_per_img)\n",
        "baseline = (sc_.sum(1, keepdims=True) - sc_) / (sc_.shape[1] - 1)\n",
        "reward = scores.reshape(batch_size, seq_per_img)\n",
        "reward = reward - baseline\n",
        "reward = reward.reshape(gen_res_size)\n",
        "reward = torch.as_tensor(reward, device=device, dtype=torch.float64)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L8ES8B8Pup3u"
      },
      "outputs": [],
      "source": [
        "batch_size = len(gt_res)\n",
        "gen_res_size = len(gen_res)*batch_size\n",
        "seq_per_img = gen_res_size // batch_size\n",
        "\n",
        "model.train()\n",
        "sample_src_tokens = torch.repeat_interleave(\n",
        "    batch['input_ids'], seq_per_img, dim=0\n",
        ")\n",
        "sample_patch_images = torch.repeat_interleave(\n",
        "        batch['patch_images'], seq_per_img, dim=0\n",
        ")\n",
        "\n",
        "len_max = 0\n",
        "for i in gen_target:\n",
        "  if i.size(1) >len_max:\n",
        "    len_max = i.size(1)\n",
        "for j in range(batch_size):\n",
        "  for i in range(seq_per_img ):\n",
        "    padding = torch.ones(len_max - len(gen_target[i][j]), dtype = torch.int64)\n",
        "    gen_padded = torch.cat((gen_target[i][j], padding))\n",
        "    if i==0 and j==0:\n",
        "      gen_prev_output_tokens = gen_padded.unsqueeze(0)\n",
        "    else:\n",
        "      gen_prev_output_tokens = torch.cat((gen_prev_output_tokens, gen_padded.unsqueeze(0)),dim=0)\n",
        "\n",
        "net_output = model(input_ids = sample_src_tokens, patch_images = sample_patch_images, decoder_input_ids = gen_prev_output_tokens)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "05JRZX7hSrRZ"
      },
      "outputs": [],
      "source": [
        "lprobs = model.get_normalized_probs(net_output, log_probs=True) # 0~1에서 로그함수적용하므로 음수행"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tzsSP3pgUsGA"
      },
      "outputs": [],
      "source": [
        "loss = -lprobs.gather(dim=-1, index=gen_prev_output_tokens.unsqueeze(-1)).squeeze() * reward.unsqueeze(-1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MbUPCCTSWpWh"
      },
      "outputs": [],
      "source": [
        "pad_mask= gen_prev_output_tokens.eq(1)\n",
        "loss.masked_fill_(pad_mask, 0.0)\n",
        "ntokens = (~pad_mask).sum()\n",
        "loss = loss.sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WInZl23DWKwd",
        "outputId": "bc1d6f8d-ec98-4300-9fd6-0d9dd2967870"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(27.9917, dtype=torch.float64, grad_fn=<SumBackward0>)\n"
          ]
        }
      ],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bnVrtuiieqdR"
      },
      "source": [
        "##SCST huggingface (not working)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rgWoawC11tF7"
      },
      "outputs": [],
      "source": [
        "from torch import nn\n",
        "from transformers import Trainer\n",
        "from pycocoevalcap.cider.cider import Cider\n",
        "\n",
        "# from torch.utils.data import DataLoader\n",
        "\n",
        "# train_dataloader = DataLoader(\n",
        "#     train_dataset,\n",
        "#     shuffle=True,\n",
        "#     batch_size=4,\n",
        "#     collate_fn=data_collator,\n",
        "# )\n",
        "\n",
        "\n",
        "class CustomTrainer(Trainer):\n",
        "    def compute_loss(self, model, inputs, return_outputs=False):\n",
        "        ## eval sample output to get reward\n",
        "        gen_target = [] #생성 토큰\n",
        "        gen_res = [] #디코드 문장\n",
        "        gt_res = [] # 정답문장\n",
        "        for batch in train_dataloader:\n",
        "\n",
        "          model.eval()\n",
        "          with torch.no_grad():\n",
        "            output = model.generate(input_ids = batch['input_ids'], patch_images = batch['patch_images'],num_beams=5, no_repeat_ngram_size=3, max_length = 40,\n",
        "                                      use_cache=False  )\n",
        "            gen_sentence = tokenizer.batch_decode(output, skip_special_tokens=True)\n",
        "            gen_target.append(output)\n",
        "            gen_res.append(gen_sentence)\n",
        "            for _ in range(4):\n",
        "              output = model.generate(input_ids = batch['input_ids'], patch_images = batch['patch_images'],temperature = 2.0,\n",
        "                                      do_sample =  True,max_length = 40,\n",
        "                                    #num_beams=5, no_repeat_ngram_size=3, \n",
        "                                      use_cache=False )\n",
        "              gen_sentence = tokenizer.batch_decode(output, skip_special_tokens=True)\n",
        "              gen_target.append(output)\n",
        "              gen_res.append(gen_sentence)\n",
        "          caption = batch['gt']\n",
        "          gt_index = batch['gt_index']\n",
        "          gt_res = caption\n",
        "        dict_gt,dict_gen = {},{}\n",
        "        for index, id in enumerate(gt_index):\n",
        "          for i in range(len(gen_res)):\n",
        "            dict_gen[str(id)+'_'+ str(i%5)] = [gen_res[i][index]]\n",
        "            dict_gt[str(id)+'_'+ str(i%5)] = [gt_res[index]]\n",
        "\n",
        "        cider,batch_cider_scores = Cider().compute_score(dict_gt, dict_gen)\n",
        "        CIDER_REWARD_WEIGHT =1\n",
        "        batch_cider_scores\n",
        "        scores = CIDER_REWARD_WEIGHT * batch_cider_scores\n",
        "\n",
        "        #scst reward\n",
        "        batch_size = len(gt_res)\n",
        "        gen_res_size = len(gen_res)*batch_size\n",
        "        seq_per_img = gen_res_size // batch_size\n",
        "\n",
        "        sc_ = scores.reshape(batch_size, seq_per_img)\n",
        "        baseline = (sc_.sum(1, keepdims=True) - sc_) / (sc_.shape[1] - 1)\n",
        "        reward = scores.reshape(batch_size, seq_per_img)\n",
        "        reward = reward - baseline\n",
        "        reward = reward.reshape(gen_res_size)\n",
        "        reward = torch.as_tensor(reward, device=device, dtype=torch.float64)\n",
        "        \n",
        "        # model.train to get net output\n",
        "        # batch_size = len(gt_res)\n",
        "        # gen_res_size = len(gen_res)*batch_size\n",
        "        # seq_per_img = gen_res_size // batch_size\n",
        "\n",
        "        model.train()\n",
        "        sample_src_tokens = torch.repeat_interleave(\n",
        "            batch['input_ids'], seq_per_img, dim=0\n",
        "        )\n",
        "        sample_patch_images = torch.repeat_interleave(\n",
        "                batch['patch_images'], seq_per_img, dim=0\n",
        "        )\n",
        "\n",
        "        len_max = 0\n",
        "        for i in gen_target:\n",
        "          if i.size(1) >len_max:\n",
        "            len_max = i.size(1)\n",
        "        for j in range(batch_size):\n",
        "          for i in range(seq_per_img ):\n",
        "            padding = torch.ones(len_max - len(gen_target[i][j]), dtype = torch.int64)\n",
        "            gen_padded = torch.cat((gen_target[i][j], padding))\n",
        "            if i==0 and j==0:\n",
        "              gen_prev_output_tokens = gen_padded.unsqueeze(0)\n",
        "            else:\n",
        "              gen_prev_output_tokens = torch.cat((gen_prev_output_tokens, gen_padded.unsqueeze(0)),dim=0)\n",
        "\n",
        "        net_output = model(input_ids = sample_src_tokens, patch_images = sample_patch_images, decoder_input_ids = gen_prev_output_tokens)\n",
        "        # compute custom loss \n",
        "        lprobs = model.get_normalized_probs(net_output, log_probs=True) \n",
        "        loss = -lprobs.gather(dim=-1, index=gen_prev_output_tokens.unsqueeze(-1)).squeeze() * reward.unsqueeze(-1)\n",
        "        pad_mask= gen_prev_output_tokens.eq(1)\n",
        "        loss.masked_fill_(pad_mask, 0.0)\n",
        "        ntokens = (~pad_mask).sum()\n",
        "        loss = loss.sum()\n",
        "        return (loss, net_output) if return_outputs else loss\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NDCFv8srYtVe"
      },
      "outputs": [],
      "source": [
        "def compute_loss(self, model, sample, reduce=True):\n",
        "    gen_target, gen_res, gt_res = self.get_generator_out(model, sample)\n",
        "    reward, scores = self.get_reward_and_scores(gen_res, gt_res, device=sample[\"target\"].device)\n",
        "    net_output, gen_target_tokens = self.get_net_output(model, sample, gen_target)\n",
        "    gen_lprobs, gen_target_tokens = self.get_lprobs_and_target(model, net_output, gen_target_tokens)\n",
        "    loss, ntokens = scst_loss(gen_lprobs, gen_target_tokens, reward, ignore_index=self.padding_idx, reduce=reduce)\n",
        "    nsentences = gen_target_tokens.size(0)\n",
        "\n",
        "    return loss, scores.sum(), ntokens, nsentences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eK4-d-vcfS1N",
        "outputId": "a470d5a2-477c-4207-b638-bf2f5e98021f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1;30;43m스트리밍 출력 내용이 길어서 마지막 5000줄이 삭제되었습니다.\u001b[0m\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4291/5000 [4:40:01<44:20,  3.75s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4292/5000 [4:40:04<44:08,  3.74s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4293/5000 [4:40:10<49:34,  4.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4294/5000 [4:40:14<49:27,  4.20s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4295/5000 [4:40:18<48:48,  4.15s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4296/5000 [4:40:22<47:05,  4.01s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4297/5000 [4:40:28<55:24,  4.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4298/5000 [4:40:32<51:30,  4.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4299/5000 [4:40:36<51:41,  4.42s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "{'loss': -1.5724, 'learning_rate': 4.758647376699033e-07, 'epoch': 1.72}\n",
            " 86% 4300/5000 [4:40:39<45:54,  3.94s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4301/5000 [4:40:43<46:57,  4.03s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4302/5000 [4:40:48<50:59,  4.38s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4303/5000 [4:40:53<50:49,  4.37s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4304/5000 [4:40:56<45:37,  3.93s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4305/5000 [4:41:00<46:47,  4.04s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4306/5000 [4:41:03<42:41,  3.69s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4307/5000 [4:41:06<40:02,  3.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4308/5000 [4:41:10<42:37,  3.70s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4309/5000 [4:41:16<50:40,  4.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4310/5000 [4:41:19<44:51,  3.90s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4311/5000 [4:41:23<45:14,  3.94s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4312/5000 [4:41:27<44:40,  3.90s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4313/5000 [4:41:29<40:14,  3.52s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4314/5000 [4:41:32<37:24,  3.27s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4315/5000 [4:41:35<36:21,  3.18s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4316/5000 [4:41:38<34:28,  3.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4317/5000 [4:41:41<35:05,  3.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4318/5000 [4:41:44<35:00,  3.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4319/5000 [4:41:47<34:22,  3.03s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4320/5000 [4:41:49<32:51,  2.90s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4321/5000 [4:41:54<37:22,  3.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4322/5000 [4:41:56<35:13,  3.12s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4323/5000 [4:41:59<35:06,  3.11s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4324/5000 [4:42:04<38:29,  3.42s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 86% 4325/5000 [4:42:07<40:09,  3.57s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4326/5000 [4:42:11<41:02,  3.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4327/5000 [4:42:16<43:13,  3.85s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4328/5000 [4:42:18<39:17,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4329/5000 [4:42:21<36:40,  3.28s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4330/5000 [4:42:24<34:21,  3.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4331/5000 [4:42:27<33:25,  3.00s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4332/5000 [4:42:30<36:23,  3.27s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4333/5000 [4:42:34<38:16,  3.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4334/5000 [4:42:37<35:59,  3.24s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4335/5000 [4:42:41<37:06,  3.35s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4336/5000 [4:42:44<38:44,  3.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4337/5000 [4:42:48<39:43,  3.59s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4338/5000 [4:42:51<36:22,  3.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4339/5000 [4:42:54<34:55,  3.17s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4340/5000 [4:42:58<38:28,  3.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4341/5000 [4:43:04<45:06,  4.11s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4342/5000 [4:43:08<44:47,  4.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4343/5000 [4:43:10<40:41,  3.72s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4344/5000 [4:43:13<37:11,  3.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4345/5000 [4:43:18<41:57,  3.84s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4346/5000 [4:43:22<43:22,  3.98s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4347/5000 [4:43:28<48:30,  4.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4348/5000 [4:43:33<51:12,  4.71s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4349/5000 [4:43:36<44:51,  4.13s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4350/5000 [4:43:39<40:00,  3.69s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4351/5000 [4:43:43<42:10,  3.90s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4352/5000 [4:43:46<37:57,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4353/5000 [4:43:50<40:16,  3.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4354/5000 [4:43:54<40:23,  3.75s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4355/5000 [4:43:58<41:32,  3.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4356/5000 [4:44:01<38:02,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4357/5000 [4:44:04<38:41,  3.61s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4358/5000 [4:44:07<35:27,  3.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4359/5000 [4:44:11<37:45,  3.53s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4360/5000 [4:44:14<35:03,  3.29s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4361/5000 [4:44:20<44:31,  4.18s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4362/5000 [4:44:26<51:00,  4.80s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4363/5000 [4:44:30<48:44,  4.59s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4364/5000 [4:44:35<47:25,  4.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4365/5000 [4:44:39<46:01,  4.35s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4366/5000 [4:44:46<55:35,  5.26s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4367/5000 [4:44:50<52:46,  5.00s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4368/5000 [4:44:53<45:22,  4.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4369/5000 [4:44:56<40:32,  3.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4370/5000 [4:44:59<36:42,  3.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4371/5000 [4:45:02<37:51,  3.61s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4372/5000 [4:45:06<38:40,  3.69s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4373/5000 [4:45:10<40:07,  3.84s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 87% 4374/5000 [4:45:15<41:19,  3.96s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4375/5000 [4:45:20<46:16,  4.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4376/5000 [4:45:24<44:13,  4.25s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4377/5000 [4:45:28<43:08,  4.15s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4378/5000 [4:45:32<41:39,  4.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4379/5000 [4:45:35<38:16,  3.70s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4380/5000 [4:45:39<38:44,  3.75s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4381/5000 [4:45:41<35:09,  3.41s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4382/5000 [4:45:46<39:24,  3.83s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4383/5000 [4:45:50<41:09,  4.00s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4384/5000 [4:45:54<40:19,  3.93s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4385/5000 [4:45:58<40:04,  3.91s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4386/5000 [4:46:03<43:47,  4.28s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4387/5000 [4:46:08<45:55,  4.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4388/5000 [4:46:13<46:41,  4.58s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4389/5000 [4:46:16<40:35,  3.99s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4390/5000 [4:46:19<40:02,  3.94s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4391/5000 [4:46:23<40:25,  3.98s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4392/5000 [4:46:27<37:56,  3.74s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4393/5000 [4:46:29<34:45,  3.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4394/5000 [4:46:33<36:07,  3.58s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4395/5000 [4:46:38<38:44,  3.84s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4396/5000 [4:46:41<38:27,  3.82s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4397/5000 [4:46:44<35:03,  3.49s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4398/5000 [4:46:47<32:11,  3.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4399/5000 [4:46:51<34:40,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "{'loss': -1.4805, 'learning_rate': 3.511175705587433e-07, 'epoch': 1.76}\n",
            " 88% 4400/5000 [4:46:57<42:59,  4.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4401/5000 [4:47:03<48:30,  4.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4402/5000 [4:47:06<42:12,  4.24s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4403/5000 [4:47:09<38:21,  3.85s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4404/5000 [4:47:12<34:29,  3.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4405/5000 [4:47:14<32:11,  3.25s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4406/5000 [4:47:22<44:13,  4.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4407/5000 [4:47:25<39:44,  4.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4408/5000 [4:47:29<39:39,  4.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4409/5000 [4:47:34<43:14,  4.39s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4410/5000 [4:47:37<38:47,  3.94s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4411/5000 [4:47:41<40:07,  4.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4412/5000 [4:47:44<36:12,  3.70s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4413/5000 [4:47:49<40:34,  4.15s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4414/5000 [4:47:52<35:58,  3.68s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4415/5000 [4:47:55<33:41,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4416/5000 [4:47:57<31:06,  3.20s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4417/5000 [4:48:00<29:46,  3.06s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4418/5000 [4:48:03<28:57,  2.99s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4419/5000 [4:48:07<32:42,  3.38s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4420/5000 [4:48:11<33:57,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4421/5000 [4:48:14<31:48,  3.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4422/5000 [4:48:16<30:17,  3.14s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4423/5000 [4:48:19<29:44,  3.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4424/5000 [4:48:24<33:08,  3.45s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 88% 4425/5000 [4:48:30<40:47,  4.26s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4426/5000 [4:48:34<38:56,  4.07s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4427/5000 [4:48:38<38:42,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4428/5000 [4:48:41<38:03,  3.99s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4429/5000 [4:48:44<34:01,  3.57s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4430/5000 [4:48:47<31:42,  3.34s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4431/5000 [4:48:50<30:09,  3.18s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4432/5000 [4:48:55<35:53,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4433/5000 [4:48:59<37:03,  3.92s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4434/5000 [4:49:02<33:01,  3.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4435/5000 [4:49:04<31:12,  3.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4436/5000 [4:49:11<39:37,  4.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4437/5000 [4:49:16<42:27,  4.52s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4438/5000 [4:49:20<40:22,  4.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4439/5000 [4:49:23<36:13,  3.87s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4440/5000 [4:49:27<37:02,  3.97s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4441/5000 [4:49:30<33:38,  3.61s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4442/5000 [4:49:34<35:20,  3.80s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4443/5000 [4:49:39<39:21,  4.24s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4444/5000 [4:49:43<38:44,  4.18s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4445/5000 [4:49:46<34:14,  3.70s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4446/5000 [4:49:50<34:43,  3.76s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4447/5000 [4:49:53<32:15,  3.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4448/5000 [4:49:56<32:57,  3.58s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4449/5000 [4:50:00<33:59,  3.70s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4450/5000 [4:50:04<34:24,  3.75s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4451/5000 [4:50:10<39:06,  4.27s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4452/5000 [4:50:12<34:46,  3.81s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4453/5000 [4:50:15<31:46,  3.49s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4454/5000 [4:50:18<29:25,  3.23s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4455/5000 [4:50:21<28:40,  3.16s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4456/5000 [4:50:23<27:03,  2.98s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4457/5000 [4:50:27<29:59,  3.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4458/5000 [4:50:31<31:11,  3.45s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4459/5000 [4:50:34<29:35,  3.28s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4460/5000 [4:50:39<35:11,  3.91s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4461/5000 [4:50:42<31:46,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4462/5000 [4:50:46<33:05,  3.69s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4463/5000 [4:50:50<34:30,  3.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4464/5000 [4:50:54<34:49,  3.90s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4465/5000 [4:50:57<31:45,  3.56s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4466/5000 [4:51:00<29:36,  3.33s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4467/5000 [4:51:04<31:24,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4468/5000 [4:51:07<28:58,  3.27s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4469/5000 [4:51:13<36:18,  4.10s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4470/5000 [4:51:18<39:29,  4.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4471/5000 [4:51:22<37:49,  4.29s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4472/5000 [4:51:25<33:34,  3.82s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4473/5000 [4:51:29<36:21,  4.14s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 89% 4474/5000 [4:51:32<32:31,  3.71s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4475/5000 [4:51:36<33:06,  3.78s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4476/5000 [4:51:40<34:15,  3.92s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4477/5000 [4:51:43<30:46,  3.53s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4478/5000 [4:51:46<28:51,  3.32s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4479/5000 [4:51:50<30:35,  3.52s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4480/5000 [4:51:53<28:21,  3.27s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4481/5000 [4:51:55<27:01,  3.12s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4482/5000 [4:51:58<26:02,  3.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4483/5000 [4:52:01<25:33,  2.97s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4484/5000 [4:52:05<27:45,  3.23s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4485/5000 [4:52:09<29:08,  3.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4486/5000 [4:52:11<27:41,  3.23s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4487/5000 [4:52:17<33:11,  3.88s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4488/5000 [4:52:20<30:30,  3.58s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4489/5000 [4:52:22<28:08,  3.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4490/5000 [4:52:26<29:36,  3.48s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4491/5000 [4:52:29<27:58,  3.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4492/5000 [4:52:32<26:16,  3.10s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4493/5000 [4:52:35<25:32,  3.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4494/5000 [4:52:37<24:42,  2.93s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4495/5000 [4:52:42<28:33,  3.39s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4496/5000 [4:52:46<29:34,  3.52s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4497/5000 [4:52:48<27:42,  3.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4498/5000 [4:52:52<28:47,  3.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4499/5000 [4:52:55<27:11,  3.26s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "{'loss': -1.2238, 'learning_rate': 2.447174185242324e-07, 'epoch': 1.8}\n",
            " 90% 4500/5000 [4:53:01<34:03,  4.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4501/5000 [4:53:05<33:54,  4.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4502/5000 [4:53:09<33:16,  4.01s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4503/5000 [4:53:12<30:28,  3.68s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4504/5000 [4:53:17<34:45,  4.20s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4505/5000 [4:53:22<36:29,  4.42s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4506/5000 [4:53:28<38:58,  4.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4507/5000 [4:53:31<34:21,  4.18s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4508/5000 [4:53:35<34:45,  4.24s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4509/5000 [4:53:40<37:01,  4.52s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4510/5000 [4:53:44<36:17,  4.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4511/5000 [4:53:47<32:39,  4.01s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4512/5000 [4:53:53<35:31,  4.37s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4513/5000 [4:53:59<40:15,  4.96s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4514/5000 [4:54:02<34:48,  4.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4515/5000 [4:54:05<31:26,  3.89s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4516/5000 [4:54:08<31:05,  3.85s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4517/5000 [4:54:11<28:17,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4518/5000 [4:54:16<32:38,  4.06s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4519/5000 [4:54:21<32:47,  4.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4520/5000 [4:54:23<29:09,  3.64s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4521/5000 [4:54:26<26:49,  3.36s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4522/5000 [4:54:28<24:57,  3.13s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4523/5000 [4:54:32<26:58,  3.39s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4524/5000 [4:54:36<27:44,  3.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 90% 4525/5000 [4:54:40<28:21,  3.58s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4526/5000 [4:54:45<32:15,  4.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4527/5000 [4:54:50<32:47,  4.16s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4528/5000 [4:54:52<29:23,  3.74s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4529/5000 [4:54:57<30:31,  3.89s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4530/5000 [4:54:59<27:43,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4531/5000 [4:55:02<25:53,  3.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4532/5000 [4:55:06<27:56,  3.58s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4533/5000 [4:55:09<25:40,  3.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4534/5000 [4:55:12<24:29,  3.15s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4535/5000 [4:55:15<23:50,  3.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4536/5000 [4:55:19<27:48,  3.60s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4537/5000 [4:55:24<29:14,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4538/5000 [4:55:26<26:42,  3.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4539/5000 [4:55:29<25:25,  3.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4540/5000 [4:55:32<24:15,  3.16s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4541/5000 [4:55:35<23:22,  3.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4542/5000 [4:55:41<30:45,  4.03s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4543/5000 [4:55:44<28:22,  3.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4544/5000 [4:55:48<28:29,  3.75s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4545/5000 [4:55:52<29:51,  3.94s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4546/5000 [4:55:57<32:13,  4.26s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4547/5000 [4:56:03<35:14,  4.67s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4548/5000 [4:56:06<30:24,  4.04s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4549/5000 [4:56:10<30:42,  4.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4550/5000 [4:56:12<27:23,  3.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4551/5000 [4:56:15<25:47,  3.45s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4552/5000 [4:56:19<27:03,  3.62s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4553/5000 [4:56:22<25:06,  3.37s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4554/5000 [4:56:26<26:42,  3.59s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4555/5000 [4:56:33<33:08,  4.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4556/5000 [4:56:38<34:58,  4.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4557/5000 [4:56:41<30:36,  4.14s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4558/5000 [4:56:44<27:22,  3.72s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4559/5000 [4:56:48<27:41,  3.77s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4560/5000 [4:56:53<31:49,  4.34s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4561/5000 [4:56:56<27:59,  3.83s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4562/5000 [4:57:01<31:01,  4.25s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4563/5000 [4:57:05<30:08,  4.14s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4564/5000 [4:57:09<29:28,  4.06s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4565/5000 [4:57:12<26:36,  3.67s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4566/5000 [4:57:14<24:33,  3.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4567/5000 [4:57:17<23:10,  3.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4568/5000 [4:57:21<24:29,  3.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4569/5000 [4:57:25<25:50,  3.60s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4570/5000 [4:57:29<27:09,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4571/5000 [4:57:32<25:03,  3.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4572/5000 [4:57:38<29:01,  4.07s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4573/5000 [4:57:40<25:54,  3.64s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 91% 4574/5000 [4:57:43<24:00,  3.38s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4575/5000 [4:57:48<28:28,  4.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4576/5000 [4:57:51<25:25,  3.60s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4577/5000 [4:57:54<23:35,  3.35s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4578/5000 [4:57:57<22:11,  3.16s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4579/5000 [4:58:01<24:21,  3.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4580/5000 [4:58:05<24:54,  3.56s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4581/5000 [4:58:07<23:07,  3.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4582/5000 [4:58:14<29:23,  4.22s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4583/5000 [4:58:18<29:41,  4.27s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4584/5000 [4:58:21<26:15,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4585/5000 [4:58:26<29:04,  4.20s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4586/5000 [4:58:28<25:40,  3.72s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4587/5000 [4:58:32<26:04,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4588/5000 [4:58:39<31:26,  4.58s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4589/5000 [4:58:41<27:23,  4.00s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4590/5000 [4:58:47<29:55,  4.38s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4591/5000 [4:58:50<26:48,  3.93s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4592/5000 [4:58:56<31:46,  4.67s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4593/5000 [4:59:00<30:52,  4.55s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4594/5000 [4:59:04<29:27,  4.35s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4595/5000 [4:59:07<26:37,  3.94s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4596/5000 [4:59:12<28:27,  4.23s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4597/5000 [4:59:19<34:47,  5.18s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4598/5000 [4:59:24<32:57,  4.92s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4599/5000 [4:59:28<31:55,  4.78s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "{'loss': -0.6691, 'learning_rate': 1.5708419435684463e-07, 'epoch': 1.84}\n",
            " 92% 4600/5000 [4:59:31<27:29,  4.12s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4601/5000 [4:59:36<29:25,  4.42s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4602/5000 [4:59:39<26:48,  4.04s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4603/5000 [4:59:42<24:30,  3.70s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4604/5000 [4:59:47<27:42,  4.20s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4605/5000 [4:59:50<24:54,  3.78s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4606/5000 [4:59:53<22:47,  3.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4607/5000 [4:59:57<24:06,  3.68s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4608/5000 [5:00:00<22:03,  3.38s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4609/5000 [5:00:04<23:13,  3.56s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4610/5000 [5:00:06<21:06,  3.25s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4611/5000 [5:00:09<20:35,  3.18s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4612/5000 [5:00:16<26:43,  4.13s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4613/5000 [5:00:18<23:56,  3.71s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4614/5000 [5:00:23<25:41,  3.99s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4615/5000 [5:00:26<23:24,  3.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4616/5000 [5:00:29<22:01,  3.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4617/5000 [5:00:32<22:23,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4618/5000 [5:00:35<20:54,  3.28s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4619/5000 [5:00:38<20:07,  3.17s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4620/5000 [5:00:41<19:22,  3.06s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4621/5000 [5:00:44<18:28,  2.92s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4622/5000 [5:00:46<18:13,  2.89s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4623/5000 [5:00:50<20:18,  3.23s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4624/5000 [5:00:54<21:23,  3.41s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 92% 4625/5000 [5:00:58<22:50,  3.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4626/5000 [5:01:02<23:00,  3.69s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4627/5000 [5:01:05<21:29,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4628/5000 [5:01:09<22:43,  3.67s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4629/5000 [5:01:14<23:48,  3.85s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4630/5000 [5:01:16<21:47,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4631/5000 [5:01:20<22:22,  3.64s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4632/5000 [5:01:23<20:41,  3.37s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4633/5000 [5:01:28<23:07,  3.78s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4634/5000 [5:01:30<21:07,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4635/5000 [5:01:37<26:49,  4.41s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4636/5000 [5:01:41<26:25,  4.36s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4637/5000 [5:01:46<27:45,  4.59s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4638/5000 [5:01:51<27:05,  4.49s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4639/5000 [5:01:55<25:58,  4.32s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4640/5000 [5:01:58<24:45,  4.13s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4641/5000 [5:02:04<28:26,  4.75s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4642/5000 [5:02:10<29:26,  4.93s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4643/5000 [5:02:13<25:47,  4.33s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4644/5000 [5:02:18<27:31,  4.64s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4645/5000 [5:02:21<23:58,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4646/5000 [5:02:25<24:25,  4.14s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4647/5000 [5:02:28<22:15,  3.78s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4648/5000 [5:02:31<20:18,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4649/5000 [5:02:36<23:23,  4.00s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4650/5000 [5:02:41<25:46,  4.42s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4651/5000 [5:02:47<27:04,  4.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4652/5000 [5:02:49<23:40,  4.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4653/5000 [5:02:53<23:34,  4.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4654/5000 [5:02:56<21:15,  3.69s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4655/5000 [5:03:00<21:59,  3.82s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4656/5000 [5:03:06<24:26,  4.26s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4657/5000 [5:03:10<23:45,  4.16s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4658/5000 [5:03:12<21:16,  3.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4659/5000 [5:03:17<22:32,  3.97s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4660/5000 [5:03:21<22:10,  3.91s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4661/5000 [5:03:25<22:44,  4.03s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4662/5000 [5:03:28<20:37,  3.66s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4663/5000 [5:03:31<19:24,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4664/5000 [5:03:35<20:00,  3.57s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4665/5000 [5:03:40<23:02,  4.13s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4666/5000 [5:03:44<22:41,  4.08s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4667/5000 [5:03:49<24:05,  4.34s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4668/5000 [5:03:53<24:02,  4.35s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4669/5000 [5:03:59<25:38,  4.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4670/5000 [5:04:01<22:16,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4671/5000 [5:04:04<20:18,  3.70s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4672/5000 [5:04:08<20:51,  3.81s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4673/5000 [5:04:12<20:33,  3.77s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 93% 4674/5000 [5:04:15<18:43,  3.45s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4675/5000 [5:04:17<17:35,  3.25s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4676/5000 [5:04:20<16:41,  3.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4677/5000 [5:04:24<17:51,  3.32s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4678/5000 [5:04:29<20:55,  3.90s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4679/5000 [5:04:33<21:08,  3.95s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4680/5000 [5:04:38<21:36,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4681/5000 [5:04:40<19:34,  3.68s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4682/5000 [5:04:43<18:01,  3.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4683/5000 [5:04:49<22:29,  4.26s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4684/5000 [5:04:54<22:24,  4.26s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4685/5000 [5:04:56<20:03,  3.82s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4686/5000 [5:05:00<19:50,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4687/5000 [5:05:06<22:38,  4.34s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4688/5000 [5:05:10<22:30,  4.33s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4689/5000 [5:05:13<19:50,  3.83s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4690/5000 [5:05:15<17:57,  3.48s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4691/5000 [5:05:18<17:12,  3.34s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4692/5000 [5:05:22<17:50,  3.48s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4693/5000 [5:05:25<16:23,  3.20s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4694/5000 [5:05:28<15:48,  3.10s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4695/5000 [5:05:35<22:08,  4.36s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4696/5000 [5:05:39<21:50,  4.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4697/5000 [5:05:44<23:05,  4.57s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4698/5000 [5:05:47<20:08,  4.00s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4699/5000 [5:05:51<19:59,  3.98s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "{'loss': -1.3495, 'learning_rate': 8.856374635655696e-08, 'epoch': 1.88}\n",
            " 94% 4700/5000 [5:05:54<18:00,  3.60s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4701/5000 [5:05:57<18:17,  3.67s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4702/5000 [5:06:05<23:53,  4.81s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4703/5000 [5:06:09<22:44,  4.59s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4704/5000 [5:06:13<21:30,  4.36s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4705/5000 [5:06:16<18:56,  3.85s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4706/5000 [5:06:20<19:31,  3.99s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4707/5000 [5:06:23<17:46,  3.64s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4708/5000 [5:06:25<16:21,  3.36s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4709/5000 [5:06:28<15:33,  3.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4710/5000 [5:06:32<16:29,  3.41s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4711/5000 [5:06:35<15:55,  3.31s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4712/5000 [5:06:38<15:09,  3.16s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4713/5000 [5:06:41<14:29,  3.03s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4714/5000 [5:06:44<14:18,  3.00s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4715/5000 [5:06:46<13:59,  2.94s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4716/5000 [5:06:49<13:44,  2.90s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4717/5000 [5:06:52<13:31,  2.87s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4718/5000 [5:06:55<13:10,  2.80s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4719/5000 [5:07:00<16:06,  3.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4720/5000 [5:07:04<17:08,  3.67s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4721/5000 [5:07:06<15:35,  3.35s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4722/5000 [5:07:12<18:15,  3.94s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4723/5000 [5:07:16<18:23,  3.98s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4724/5000 [5:07:19<16:35,  3.61s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 94% 4725/5000 [5:07:21<15:30,  3.38s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4726/5000 [5:07:26<17:19,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4727/5000 [5:07:30<17:33,  3.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4728/5000 [5:07:33<16:05,  3.55s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4729/5000 [5:07:37<16:14,  3.60s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4730/5000 [5:07:41<16:32,  3.68s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4731/5000 [5:07:45<17:19,  3.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4732/5000 [5:07:48<15:54,  3.56s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4733/5000 [5:07:52<16:43,  3.76s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4734/5000 [5:07:56<17:41,  3.99s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4735/5000 [5:08:01<18:53,  4.28s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4736/5000 [5:08:04<16:38,  3.78s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4737/5000 [5:08:07<15:16,  3.48s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4738/5000 [5:08:09<14:05,  3.23s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4739/5000 [5:08:14<15:24,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4740/5000 [5:08:16<14:12,  3.28s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4741/5000 [5:08:20<14:42,  3.41s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4742/5000 [5:08:24<15:48,  3.67s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4743/5000 [5:08:28<16:04,  3.75s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4744/5000 [5:08:31<14:41,  3.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4745/5000 [5:08:34<13:38,  3.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4746/5000 [5:08:36<12:53,  3.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4747/5000 [5:08:39<12:45,  3.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4748/5000 [5:08:42<12:17,  2.93s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4749/5000 [5:08:45<11:51,  2.84s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4750/5000 [5:08:48<12:59,  3.12s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4751/5000 [5:08:53<14:08,  3.41s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4752/5000 [5:08:55<13:15,  3.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4753/5000 [5:09:00<15:17,  3.72s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4754/5000 [5:09:03<13:56,  3.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4755/5000 [5:09:07<14:38,  3.59s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4756/5000 [5:09:09<13:22,  3.29s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4757/5000 [5:09:12<12:31,  3.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4758/5000 [5:09:15<11:54,  2.95s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4759/5000 [5:09:20<14:58,  3.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4760/5000 [5:09:25<15:32,  3.89s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4761/5000 [5:09:27<13:58,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4762/5000 [5:09:32<15:23,  3.88s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4763/5000 [5:09:35<14:12,  3.60s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4764/5000 [5:09:39<14:22,  3.66s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4765/5000 [5:09:44<16:12,  4.14s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4766/5000 [5:09:48<16:03,  4.12s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4767/5000 [5:09:51<14:38,  3.77s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4768/5000 [5:09:54<13:26,  3.48s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4769/5000 [5:09:59<15:38,  4.06s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4770/5000 [5:10:03<15:20,  4.00s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4771/5000 [5:10:09<17:00,  4.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4772/5000 [5:10:14<17:51,  4.70s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4773/5000 [5:10:18<16:51,  4.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 95% 4774/5000 [5:10:22<16:20,  4.34s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4775/5000 [5:10:28<18:42,  4.99s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4776/5000 [5:10:31<15:55,  4.27s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4777/5000 [5:10:35<15:28,  4.17s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4778/5000 [5:10:40<16:42,  4.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4779/5000 [5:10:44<16:03,  4.36s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4780/5000 [5:10:47<14:11,  3.87s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4781/5000 [5:10:51<14:26,  3.95s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4782/5000 [5:10:54<12:57,  3.57s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4783/5000 [5:10:58<13:48,  3.82s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4784/5000 [5:11:03<14:57,  4.16s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4785/5000 [5:11:08<16:06,  4.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4786/5000 [5:11:11<14:09,  3.97s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4787/5000 [5:11:14<12:53,  3.63s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4788/5000 [5:11:17<11:52,  3.36s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4789/5000 [5:11:20<12:20,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4790/5000 [5:11:24<12:40,  3.62s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4791/5000 [5:11:27<11:58,  3.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4792/5000 [5:11:31<12:11,  3.52s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4793/5000 [5:11:34<11:08,  3.23s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4794/5000 [5:11:39<13:15,  3.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4795/5000 [5:11:42<12:18,  3.60s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4796/5000 [5:11:46<12:56,  3.81s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4797/5000 [5:11:49<11:50,  3.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4798/5000 [5:11:54<13:41,  4.07s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4799/5000 [5:11:58<13:34,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "{'loss': -0.6268, 'learning_rate': 3.9426493427611177e-08, 'epoch': 1.92}\n",
            " 96% 4800/5000 [5:12:01<12:10,  3.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4801/5000 [5:12:05<12:47,  3.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4802/5000 [5:12:08<11:34,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4803/5000 [5:12:13<12:26,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4804/5000 [5:12:19<14:38,  4.48s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4805/5000 [5:12:22<13:46,  4.24s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4806/5000 [5:12:27<13:43,  4.24s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4807/5000 [5:12:29<12:12,  3.80s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4808/5000 [5:12:32<11:00,  3.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4809/5000 [5:12:35<10:15,  3.23s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4810/5000 [5:12:39<11:13,  3.55s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4811/5000 [5:12:44<12:57,  4.11s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4812/5000 [5:12:49<13:01,  4.16s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4813/5000 [5:12:53<13:02,  4.19s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4814/5000 [5:12:58<14:00,  4.52s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4815/5000 [5:13:01<12:25,  4.03s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4816/5000 [5:13:07<14:04,  4.59s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4817/5000 [5:13:13<15:36,  5.12s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4818/5000 [5:13:18<14:40,  4.84s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4819/5000 [5:13:22<14:11,  4.71s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4820/5000 [5:13:26<13:15,  4.42s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4821/5000 [5:13:30<12:41,  4.25s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4822/5000 [5:13:33<11:36,  3.91s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4823/5000 [5:13:37<11:54,  4.03s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4824/5000 [5:13:41<11:50,  4.04s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 96% 4825/5000 [5:13:47<13:47,  4.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4826/5000 [5:13:51<12:17,  4.24s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4827/5000 [5:13:55<12:01,  4.17s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4828/5000 [5:13:59<12:01,  4.19s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4829/5000 [5:14:04<12:30,  4.39s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4830/5000 [5:14:08<12:07,  4.28s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4831/5000 [5:14:13<13:01,  4.62s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4832/5000 [5:14:17<12:39,  4.52s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4833/5000 [5:14:22<12:19,  4.43s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4834/5000 [5:14:24<10:51,  3.92s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4835/5000 [5:14:29<11:45,  4.28s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4836/5000 [5:14:32<10:21,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4837/5000 [5:14:36<10:37,  3.91s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4838/5000 [5:14:41<11:28,  4.25s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4839/5000 [5:14:44<10:29,  3.91s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4840/5000 [5:14:49<10:38,  3.99s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4841/5000 [5:14:51<09:34,  3.61s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4842/5000 [5:14:54<08:48,  3.34s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4843/5000 [5:15:00<10:37,  4.06s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4844/5000 [5:15:03<09:33,  3.68s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4845/5000 [5:15:07<09:57,  3.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4846/5000 [5:15:09<08:55,  3.48s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4847/5000 [5:15:13<08:50,  3.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4848/5000 [5:15:17<09:26,  3.72s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4849/5000 [5:15:21<09:26,  3.75s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4850/5000 [5:15:24<08:40,  3.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4851/5000 [5:15:27<08:15,  3.32s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4852/5000 [5:15:29<07:41,  3.12s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4853/5000 [5:15:33<08:03,  3.29s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4854/5000 [5:15:37<08:39,  3.56s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4855/5000 [5:15:40<08:04,  3.34s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4856/5000 [5:15:46<10:04,  4.20s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4857/5000 [5:15:49<08:59,  3.77s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4858/5000 [5:15:53<09:16,  3.92s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4859/5000 [5:15:58<09:49,  4.18s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4860/5000 [5:16:02<09:27,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4861/5000 [5:16:06<09:29,  4.10s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4862/5000 [5:16:10<09:29,  4.12s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4863/5000 [5:16:14<09:23,  4.11s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4864/5000 [5:16:18<09:04,  4.00s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4865/5000 [5:16:22<08:36,  3.82s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4866/5000 [5:16:29<10:47,  4.83s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4867/5000 [5:16:32<09:19,  4.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4868/5000 [5:16:34<08:12,  3.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4869/5000 [5:16:37<07:29,  3.43s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4870/5000 [5:16:40<07:05,  3.27s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4871/5000 [5:16:44<07:48,  3.63s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4872/5000 [5:16:47<07:11,  3.37s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4873/5000 [5:16:50<06:40,  3.15s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 97% 4874/5000 [5:16:52<06:22,  3.03s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4875/5000 [5:16:57<07:12,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4876/5000 [5:17:01<07:21,  3.56s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4877/5000 [5:17:03<06:46,  3.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4878/5000 [5:17:07<07:02,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4879/5000 [5:17:10<06:36,  3.28s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4880/5000 [5:17:16<08:24,  4.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4881/5000 [5:17:21<08:17,  4.18s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4882/5000 [5:17:25<08:15,  4.20s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4883/5000 [5:17:29<08:03,  4.13s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4884/5000 [5:17:31<07:04,  3.66s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4885/5000 [5:17:34<06:24,  3.34s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4886/5000 [5:17:37<05:59,  3.15s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4887/5000 [5:17:40<05:49,  3.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4888/5000 [5:17:42<05:32,  2.97s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4889/5000 [5:17:45<05:20,  2.89s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4890/5000 [5:17:48<05:10,  2.82s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4891/5000 [5:17:51<05:14,  2.88s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4892/5000 [5:17:53<05:02,  2.80s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4893/5000 [5:17:59<06:19,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4894/5000 [5:18:01<05:51,  3.32s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4895/5000 [5:18:04<05:40,  3.24s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4896/5000 [5:18:07<05:18,  3.07s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4897/5000 [5:18:13<06:56,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4898/5000 [5:18:17<06:49,  4.01s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4899/5000 [5:18:20<06:14,  3.71s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "{'loss': -0.5104, 'learning_rate': 9.866357858642206e-09, 'epoch': 1.96}\n",
            " 98% 4900/5000 [5:18:23<05:44,  3.44s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4901/5000 [5:18:27<06:06,  3.71s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4902/5000 [5:18:30<05:40,  3.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4903/5000 [5:18:33<05:11,  3.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4904/5000 [5:18:36<04:51,  3.04s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4905/5000 [5:18:38<04:40,  2.96s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4906/5000 [5:18:41<04:39,  2.98s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4907/5000 [5:18:44<04:27,  2.88s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4908/5000 [5:18:49<05:21,  3.49s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4909/5000 [5:18:54<06:08,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4910/5000 [5:18:57<05:31,  3.69s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4911/5000 [5:19:01<05:28,  3.69s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4912/5000 [5:19:05<05:39,  3.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4913/5000 [5:19:08<05:05,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4914/5000 [5:19:13<05:51,  4.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4915/5000 [5:19:18<06:05,  4.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4916/5000 [5:19:21<05:23,  3.85s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4917/5000 [5:19:25<05:18,  3.83s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4918/5000 [5:19:27<04:50,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4919/5000 [5:19:33<05:27,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4920/5000 [5:19:35<04:51,  3.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4921/5000 [5:19:39<04:49,  3.66s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4922/5000 [5:19:42<04:25,  3.41s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4923/5000 [5:19:46<04:32,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4924/5000 [5:19:49<04:10,  3.30s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 98% 4925/5000 [5:19:51<03:55,  3.15s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4926/5000 [5:19:57<04:46,  3.87s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4927/5000 [5:20:01<04:42,  3.87s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4928/5000 [5:20:03<04:12,  3.50s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4929/5000 [5:20:10<05:16,  4.45s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4930/5000 [5:20:15<05:25,  4.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4931/5000 [5:20:20<05:29,  4.78s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4932/5000 [5:20:25<05:30,  4.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4933/5000 [5:20:28<04:41,  4.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4934/5000 [5:20:34<05:20,  4.86s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4935/5000 [5:20:39<05:02,  4.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4936/5000 [5:20:43<04:49,  4.53s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4937/5000 [5:20:47<04:39,  4.43s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4938/5000 [5:20:50<04:06,  3.98s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4939/5000 [5:20:53<03:39,  3.60s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4940/5000 [5:20:59<04:21,  4.35s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4941/5000 [5:21:03<04:07,  4.19s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4942/5000 [5:21:05<03:38,  3.76s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4943/5000 [5:21:08<03:17,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4944/5000 [5:21:13<03:41,  3.95s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4945/5000 [5:21:16<03:22,  3.69s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4946/5000 [5:21:19<03:06,  3.46s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4947/5000 [5:21:23<03:09,  3.58s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4948/5000 [5:21:26<02:54,  3.35s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4949/5000 [5:21:30<02:58,  3.51s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4950/5000 [5:21:33<02:47,  3.36s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4951/5000 [5:21:37<02:51,  3.49s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4952/5000 [5:21:39<02:34,  3.22s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4953/5000 [5:21:42<02:22,  3.04s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4954/5000 [5:21:47<02:47,  3.65s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4955/5000 [5:21:49<02:30,  3.35s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4956/5000 [5:21:55<02:51,  3.90s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4957/5000 [5:21:57<02:32,  3.54s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4958/5000 [5:22:01<02:33,  3.66s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4959/5000 [5:22:06<02:37,  3.84s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4960/5000 [5:22:08<02:19,  3.49s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4961/5000 [5:22:12<02:21,  3.63s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4962/5000 [5:22:15<02:09,  3.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4963/5000 [5:22:18<01:58,  3.20s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4964/5000 [5:22:22<02:02,  3.40s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4965/5000 [5:22:24<01:52,  3.21s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4966/5000 [5:22:27<01:45,  3.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4967/5000 [5:22:31<01:52,  3.39s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4968/5000 [5:22:34<01:42,  3.19s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4969/5000 [5:22:40<02:09,  4.16s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4970/5000 [5:22:43<01:53,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4971/5000 [5:22:46<01:40,  3.47s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4972/5000 [5:22:51<01:52,  4.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4973/5000 [5:22:54<01:37,  3.60s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            " 99% 4974/5000 [5:22:58<01:40,  3.85s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4975/5000 [5:23:03<01:39,  3.96s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4976/5000 [5:23:06<01:33,  3.89s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4977/5000 [5:23:11<01:32,  4.02s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4978/5000 [5:23:15<01:29,  4.07s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4979/5000 [5:23:17<01:16,  3.62s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4980/5000 [5:23:21<01:13,  3.67s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4981/5000 [5:23:25<01:10,  3.72s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4982/5000 [5:23:29<01:07,  3.77s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4983/5000 [5:23:33<01:06,  3.92s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4984/5000 [5:23:36<00:56,  3.55s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4985/5000 [5:23:40<00:56,  3.78s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4986/5000 [5:23:43<00:49,  3.53s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4987/5000 [5:23:49<00:52,  4.07s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4988/5000 [5:23:53<00:48,  4.07s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4989/5000 [5:23:57<00:45,  4.14s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4990/5000 [5:24:00<00:37,  3.79s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4991/5000 [5:24:05<00:38,  4.26s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4992/5000 [5:24:08<00:29,  3.74s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4993/5000 [5:24:11<00:24,  3.45s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4994/5000 [5:24:16<00:24,  4.05s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4995/5000 [5:24:20<00:20,  4.11s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4996/5000 [5:24:26<00:18,  4.68s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4997/5000 [5:24:29<00:12,  4.09s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4998/5000 [5:24:32<00:07,  3.73s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "100% 4999/5000 [5:24:36<00:03,  3.87s/it]./vocab\n",
            "<super: <class 'OFATokenizer'>, <OFATokenizer object>>\n",
            "loading file ./vocab/vocab.json\n",
            "loading file ./vocab/merges.txt\n",
            "loading file ./vocab/added_tokens.json\n",
            "loading file ./vocab/special_tokens_map.json\n",
            "loading file ./vocab/tokenizer_config.json\n",
            "{'loss': -1.4745, 'learning_rate': 0.0, 'epoch': 2.0}\n",
            "100% 5000/5000 [5:24:39<00:00,  3.54s/it]Saving model checkpoint to ../drive/MyDrive/NICE/model/0429_scst/checkpoint-5000\n",
            "Configuration saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-5000/config.json\n",
            "Model weights saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-5000/pytorch_model.bin\n",
            "tokenizer config file saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-5000/tokenizer_config.json\n",
            "Special tokens file saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-5000/special_tokens_map.json\n",
            "added tokens file saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-5000/added_tokens.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 19511.661, 'train_samples_per_second': 0.513, 'train_steps_per_second': 0.256, 'train_loss': -25.904842650699617, 'epoch': 2.0}\n",
            "100% 5000/5000 [5:25:11<00:00,  3.90s/it]\n",
            "***** train metrics *****\n",
            "  epoch                    =        2.0\n",
            "  train_loss               =   -25.9048\n",
            "  train_runtime            = 5:25:11.66\n",
            "  train_samples_per_second =      0.513\n",
            "  train_steps_per_second   =      0.256\n",
            "Saving model checkpoint to ../drive/MyDrive/NICE/model/0429_scst/checkpoint-final\n",
            "Configuration saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-final/config.json\n",
            "Model weights saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-final/pytorch_model.bin\n",
            "tokenizer config file saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-final/tokenizer_config.json\n",
            "Special tokens file saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-final/special_tokens_map.json\n",
            "added tokens file saved in ../drive/MyDrive/NICE/model/0429_scst/checkpoint-final/added_tokens.json\n"
          ]
        }
      ],
      "source": [
        "!CUDA_VISIBLE_DEVICES=0 python train.py --train_args_file train_args/train_ofa.json "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gfhRJnJxYKwE"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "\n",
        "def forward(self, model, sample, update_num=0, reduce=True):\n",
        "    \"\"\"Compute the loss for the given sample.\n",
        "    Returns a tuple with three elements:\n",
        "    1) the loss\n",
        "    2) the sample size, which is used as the denominator for the gradient\n",
        "    3) logging outputs to display while training\n",
        "    \"\"\"\n",
        "    loss, score, ntokens, nsentences = self.compute_loss(model, sample, reduce=reduce)\n",
        "\n",
        "    sample_size = (\n",
        "        nsentences if self.sentence_avg else ntokens\n",
        "    )\n",
        "    logging_output = {\n",
        "        \"loss\": loss.data,\n",
        "        \"score\": score,\n",
        "        \"ntokens\": ntokens,\n",
        "        \"nsentences\": nsentences,\n",
        "        \"sample_size\": sample_size,\n",
        "    }\n",
        "    return loss, sample_size, logging_output\n",
        "    \n",
        "def compute_loss(self, model, sample, reduce=True):\n",
        "    gen_target, gen_res, gt_res = self.get_generator_out(model, sample)\n",
        "    reward, scores = self.get_reward_and_scores(gen_res, gt_res, device=sample[\"target\"].device)\n",
        "    net_output, gen_target_tokens = self.get_net_output(model, sample, gen_target)\n",
        "    gen_lprobs, gen_target_tokens = self.get_lprobs_and_target(model, net_output, gen_target_tokens)\n",
        "    loss, ntokens = scst_loss(gen_lprobs, gen_target_tokens, reward, ignore_index=self.padding_idx, reduce=reduce)\n",
        "    nsentences = gen_target_tokens.size(0)\n",
        "\n",
        "    return loss, scores.sum(), ntokens, nsentences\n",
        "\n",
        "def get_generator_out(self, model, sample): #1\n",
        "    def decode(toks):\n",
        "        hypo = toks.int().cpu()\n",
        "        hypo_str = self.task.tgt_dict.string(hypo)\n",
        "        hypo_str = self.task.bpe.decode(hypo_str).strip()\n",
        "        return hypo, hypo_str\n",
        "\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        self.task.scst_generator.model.eval()\n",
        "        gen_out = self.task.scst_generator.generate([model], sample)\n",
        "\n",
        "    gen_target = []\n",
        "    gen_res = []\n",
        "    gt_res = []\n",
        "    for i in range(len(gen_out)):\n",
        "        for j in range(len(gen_out[i])):\n",
        "            hypo, hypo_str = decode(gen_out[i][j][\"tokens\"])\n",
        "            gen_target.append(hypo)\n",
        "            gen_res.append(hypo_str)\n",
        "        gt_res.append(\n",
        "            decode(utils.strip_pad(sample[\"target\"][i], self.padding_idx))[1].split('&&')\n",
        "        )\n",
        "\n",
        "    return gen_target, gen_res, gt_res\n",
        "\n",
        "def get_reward_and_scores(self, gen_res, gt_res, device): #2\n",
        "    batch_size = len(gt_res)\n",
        "    gen_res_size = len(gen_res)\n",
        "    seq_per_img = gen_res_size // batch_size\n",
        "\n",
        "    gt_idx = [i // seq_per_img for i in range(gen_res_size)]\n",
        "    scores = self._calculate_eval_scores(gen_res, gt_idx, gt_res) ##get ciderscore\n",
        "    sc_ = scores.reshape(batch_size, seq_per_img)\n",
        "    baseline = (sc_.sum(1, keepdims=True) - sc_) / (sc_.shape[1] - 1)\n",
        "    # sample - baseline\n",
        "    reward = scores.reshape(batch_size, seq_per_img)\n",
        "    reward = reward - baseline\n",
        "    reward = reward.reshape(gen_res_size)\n",
        "    reward = torch.as_tensor(reward, device=device, dtype=torch.float64)\n",
        "\n",
        "    return reward, scores\n",
        "\n",
        "def get_net_output(self, model, sample, gen_target): #3\n",
        "    def merge(sample_list, eos=self.task.tgt_dict.eos(), move_eos_to_beginning=False):\n",
        "        return data_utils.collate_tokens(\n",
        "            sample_list,\n",
        "            pad_idx=self.padding_idx,\n",
        "            eos_idx=eos,\n",
        "            left_pad=False,\n",
        "            move_eos_to_beginning=move_eos_to_beginning,\n",
        "        )\n",
        "\n",
        "    batch_size = len(sample[\"target\"])\n",
        "    gen_target_size = len(gen_target)\n",
        "    seq_per_img = gen_target_size // batch_size\n",
        "\n",
        "    model.train()\n",
        "    sample_src_tokens = torch.repeat_interleave(\n",
        "        sample['net_input']['src_tokens'], seq_per_img, dim=0\n",
        "    )\n",
        "    sample_src_lengths = torch.repeat_interleave(\n",
        "        sample['net_input']['src_lengths'], seq_per_img, dim=0\n",
        "    )\n",
        "    sample_patch_images = torch.repeat_interleave(\n",
        "        sample['net_input']['patch_images'], seq_per_img, dim=0\n",
        "    )\n",
        "    sample_patch_masks = torch.repeat_interleave(\n",
        "        sample['net_input']['patch_masks'], seq_per_img, dim=0\n",
        "    )\n",
        "    gen_prev_output_tokens = torch.as_tensor(\n",
        "        merge(gen_target, eos=self.task.tgt_dict.bos(), move_eos_to_beginning=True),\n",
        "        device=sample[\"target\"].device, dtype=torch.int64\n",
        "    )\n",
        "    gen_target_tokens = torch.as_tensor(\n",
        "        merge(gen_target), device=sample[\"target\"].device, dtype=torch.int64\n",
        "    )\n",
        "    net_output = model(\n",
        "        src_tokens=sample_src_tokens, src_lengths=sample_src_lengths,\n",
        "        patch_images=sample_patch_images, patch_masks=sample_patch_masks,\n",
        "        prev_output_tokens=gen_prev_output_tokens\n",
        "    )\n",
        "\n",
        "    return net_output, gen_target_tokens\n",
        "\n",
        "def get_lprobs_and_target(self, model, net_output, gen_target): #4\n",
        "    if self.constraint_start is not None and self.constraint_end is not None:\n",
        "        net_output[0][:, :, 4:self.constraint_start] = -math.inf\n",
        "        net_output[0][:, :, self.constraint_end:] = -math.inf\n",
        "    lprobs = model.get_normalized_probs(net_output, log_probs=True)\n",
        "    if self.ignore_prefix_size > 0:\n",
        "        if getattr(lprobs, \"batch_first\", False):\n",
        "            lprobs = lprobs[:, self.ignore_prefix_size :, :].contiguous()\n",
        "            gen_target = gen_target[:, self.ignore_prefix_size :].contiguous()\n",
        "        else:\n",
        "            lprobs = lprobs[self.ignore_prefix_size :, :, :].contiguous()\n",
        "            gen_target = gen_target[self.ignore_prefix_size :, :].contiguous()\n",
        "    return lprobs, gen_target\n",
        "\n",
        "  \n",
        "def scst_loss(lprobs, target, reward, ignore_index=None, reduce=True): #5\n",
        "    loss = -lprobs.gather(dim=-1, index=target.unsqueeze(-1)).squeeze() * reward.unsqueeze(-1)\n",
        "    if ignore_index is not None:\n",
        "        pad_mask = target.eq(ignore_index)\n",
        "        loss.masked_fill_(pad_mask, 0.0)\n",
        "        ntokens = (~pad_mask).sum()\n",
        "    else:\n",
        "        loss = loss.squeeze(-1)\n",
        "        ntokens = target.numel()\n",
        "    if reduce:\n",
        "        loss = loss.sum()\n",
        "    return loss, ntokens\n",
        "\n",
        "\n",
        "\n",
        "  \n",
        "#scst_loss 보니 reward는 scalar로 나오나? sequence 전반에 대한 sum?"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "1jjaSayaEiVa",
        "_Ajm6O_bYoj_",
        "bnVrtuiieqdR"
      ],
      "machine_shape": "hm",
      "provenance": [],
      "gpuType": "A100",
      "gpuClass": "premium",
      "authorship_tag": "ABX9TyMFTX7/ayXN07aOyVlnuoIY",
      "include_colab_link": true
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ba59384858e448c1a77dd7046329e936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_114b79bfbcc04fbe88f022d3d6a16950",
              "IPY_MODEL_ddfb571f11e04eaa88762379f53a17ab",
              "IPY_MODEL_678b7fb362ad4ab88943aeafcc1ac9aa"
            ],
            "layout": "IPY_MODEL_576c4643c5584d8cab69b2fc44993d75"
          }
        },
        "114b79bfbcc04fbe88f022d3d6a16950": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9cd3bc45c234493eaac60627ac333903",
            "placeholder": "​",
            "style": "IPY_MODEL_b149199c650d4b709a68c8a136876b41",
            "value": "Downloading: 100%"
          }
        },
        "ddfb571f11e04eaa88762379f53a17ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd001b788ada4bb79b0e41b53590f020",
            "max": 898823,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ccb6c43e7edb4308808839d66c930286",
            "value": 898823
          }
        },
        "678b7fb362ad4ab88943aeafcc1ac9aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae0ecfca6d48481788076a9ee1a1a8c1",
            "placeholder": "​",
            "style": "IPY_MODEL_42959e14402048b6ad4779379566235a",
            "value": " 878k/878k [00:00&lt;00:00, 1.36MB/s]"
          }
        },
        "576c4643c5584d8cab69b2fc44993d75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9cd3bc45c234493eaac60627ac333903": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b149199c650d4b709a68c8a136876b41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cd001b788ada4bb79b0e41b53590f020": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ccb6c43e7edb4308808839d66c930286": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ae0ecfca6d48481788076a9ee1a1a8c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42959e14402048b6ad4779379566235a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cffdd4e4e7974cb6b89a55a6f020e0bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_079c3986fb77465eb3ae25e29d1ac74b",
              "IPY_MODEL_bdf99afe326347b88b7e243a56e9c8c3",
              "IPY_MODEL_9efc6fc7a9504e38895d02237cfa9865"
            ],
            "layout": "IPY_MODEL_fa559e74e3dd45c7b99e8df890c0b1f8"
          }
        },
        "079c3986fb77465eb3ae25e29d1ac74b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f67ae791b94a4080ae64d929bfd540e2",
            "placeholder": "​",
            "style": "IPY_MODEL_34bdb6732cbd4aa29a6a1f09d698127e",
            "value": "Downloading: 100%"
          }
        },
        "bdf99afe326347b88b7e243a56e9c8c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b9ec3753f1564919b55a870c7f214848",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7e543fdc5e8d4961ae779e6bcae81b67",
            "value": 456318
          }
        },
        "9efc6fc7a9504e38895d02237cfa9865": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12a894a7eb8c49bca26abe13ba73cd94",
            "placeholder": "​",
            "style": "IPY_MODEL_8e6bf629eec745d28cb1b4a571cb8018",
            "value": " 446k/446k [00:00&lt;00:00, 1.24MB/s]"
          }
        },
        "fa559e74e3dd45c7b99e8df890c0b1f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f67ae791b94a4080ae64d929bfd540e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "34bdb6732cbd4aa29a6a1f09d698127e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b9ec3753f1564919b55a870c7f214848": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e543fdc5e8d4961ae779e6bcae81b67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "12a894a7eb8c49bca26abe13ba73cd94": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e6bf629eec745d28cb1b4a571cb8018": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "928f936b952b4a2fb616c430efa3348c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0a8bdc7012b94e3dbd70c277bf665c4e",
              "IPY_MODEL_654cf185b7944a8da5c675d958f55bff",
              "IPY_MODEL_1f447c78b42146e788a044dfacca0e32"
            ],
            "layout": "IPY_MODEL_788cbf7ba1e640bbbcc06cf2f3b594f8"
          }
        },
        "0a8bdc7012b94e3dbd70c277bf665c4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_29e50b0d2b174bfb8947a2d7d63d3090",
            "placeholder": "​",
            "style": "IPY_MODEL_fe3a0da63960400181974dddb320e552",
            "value": "Downloading: 100%"
          }
        },
        "654cf185b7944a8da5c675d958f55bff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b88260c780434d5a9916dc269350a61a",
            "max": 1396,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_80f93cb699984e92b0101c6bda2ba41b",
            "value": 1396
          }
        },
        "1f447c78b42146e788a044dfacca0e32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0f032ee70e534ca4b9caff416c8bc479",
            "placeholder": "​",
            "style": "IPY_MODEL_d43ecd3bc74d4756b63fe7df079c6db7",
            "value": " 1.36k/1.36k [00:00&lt;00:00, 118kB/s]"
          }
        },
        "788cbf7ba1e640bbbcc06cf2f3b594f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "29e50b0d2b174bfb8947a2d7d63d3090": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe3a0da63960400181974dddb320e552": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b88260c780434d5a9916dc269350a61a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80f93cb699984e92b0101c6bda2ba41b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0f032ee70e534ca4b9caff416c8bc479": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d43ecd3bc74d4756b63fe7df079c6db7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}