gpt_2_small.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5zbNWDt8DH0f"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "WmlHag8Ewl9i"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "GVZ5ahwyxk9Y"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.nn import functional as F\n",
        "from dataclasses import dataclass\n",
        "\n",
        "@dataclass\n",
        "class GPTLanguageModelConfig:\n",
        "  block_size: int = 1024,\n",
        "  vocab_size: int = 50_257,\n",
        "  n_embed: int = 768,\n",
        "  n_heads: int = 12,\n",
        "  n_blocks: int = 12,\n",
        "  dropout_rate: int = 0.2,\n",
        "  device: str = \"cuda\"\n",
        "\n",
        "class Head(nn.Module):\n",
        "    \"\"\"A single self-attention head\"\"\"\n",
        "\n",
        "    def __init__(self, config):\n",
        "        super().__init__()\n",
        "        head_size = config.n_embed // config.n_heads\n",
        "\n",
        "        self.key = nn.Linear(config.n_embed, head_size, bias=False)\n",
        "        self.query = nn.Linear(config.n_embed, head_size, bias=False)\n",
        "        self.value = nn.Linear(config.n_embed, head_size, bias=False)\n",
        "        self.register_buffer('tril', torch.tril(torch.ones(config.block_size, config.block_size, dtype=torch.bool)))\n",
        "        self.dropout = nn.Dropout(config.dropout_rate)\n",
        "\n",
        "    def forward(self, x):\n",
        "        B, T, C = x.shape # batch size, block size, n_embed\n",
        "        k = self.key(x) # (B, T, head_size)\n",
        "        q = self.query(x) # (B, T, head_size)\n",
        "\n",
        "        wei = q @ k.transpose(-2, -1) * (C ** -0.5) # (B, T, T)\n",
        "        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))\n",
        "        wei = F.softmax(wei, dim=-1)\n",
        "\n",
        "        wei = self.dropout(wei)\n",
        "\n",
        "        v = self.value(x) # (B, T, head_size)\n",
        "        out = wei @ v # (B, T, head_size)\n",
        "        return out\n",
        "\n",
        "class MultiHeadAttention(nn.Module):\n",
        "    \"\"\"Multiple heads of self-attention in parallel\"\"\"\n",
        "\n",
        "    def __init__(self, config):\n",
        "        super().__init__()\n",
        "\n",
        "        self.heads = nn.ModuleList([Head(config) for _ in range(config.n_heads)])\n",
        "        self.proj = nn.Linear(config.n_embed, config.n_embed) # NOTE: in the paper dims say n_heads * head_size, which is same as n_embed in our case\n",
        "        self.dropout = nn.Dropout(config.dropout_rate)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = torch.cat([h(x) for h in self.heads], dim=-1) # we concat on dim -1 because we want a (B, T, n_heads*head_size) tensor so we need to concat on the final dim, this ends up being same as (B, T, C) though because we configured head_size to be n_embed / num_heads\n",
        "        proj = self.proj(out) # final projection, W^o in the paper\n",
        "        return self.dropout(proj)\n",
        "\n",
        "class FeedForward(nn.Module):\n",
        "    \"\"\"A simple feed-forward module\"\"\"\n",
        "\n",
        "    def __init__(self, config):\n",
        "        super().__init__()\n",
        "\n",
        "        # the paper denotes \"two linear transformations with a ReLU activation in between\"\n",
        "        # also note the 4x expansion in middle is due to detail in paper in the feedforward section\n",
        "        self.net = nn.Sequential(\n",
        "            nn.Linear(config.n_embed, 4 * config.n_embed),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(4 * config.n_embed, config.n_embed),\n",
        "            nn.Dropout(config.dropout_rate)\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.net(x)\n",
        "\n",
        "class TransformerBlock(nn.Module):\n",
        "    \"\"\"Transformer block\"\"\"\n",
        "\n",
        "    def __init__(self, config):\n",
        "        super().__init__()\n",
        "\n",
        "        self.layernorm1 = nn.LayerNorm(config.n_embed)\n",
        "        self.sa = MultiHeadAttention(config)\n",
        "        self.layernorm2 = nn.LayerNorm(config.n_embed)\n",
        "        self.ffwd = FeedForward(config)\n",
        "\n",
        "    def forward(self, x):\n",
        "        # add \"x +\" as part of residual connection (helps with vanishing gradients in deep network)\n",
        "        # also note layernorm now comes before self-attention and feedforward despite paper saying after\n",
        "        x_norm1 = self.layernorm1(x)\n",
        "        x = x + self.sa(x_norm1)\n",
        "        x_norm2 = self.layernorm2(x)\n",
        "        x = x + self.ffwd(x_norm2)\n",
        "        return x\n",
        "\n",
        "class GPTLanguageModel(nn.Module):\n",
        "    def __init__(self, config: GPTLanguageModelConfig):\n",
        "        super().__init__()\n",
        "        self.config = config\n",
        "\n",
        "        # each token directly reads off the logits for the next token from a lookup table\n",
        "        # NOTE: the embedding layer has vocab_size keys each of which has a n_embed dim value, nn.Embedding is basically just a lookup table\n",
        "        self.token_embedding_table = nn.Embedding(config.vocab_size, config.n_embed) # token embeddings, takes (B,T) and outputs (B,T,C) where C is embedding size\n",
        "        self.position_embedding_table = nn.Embedding(config.block_size, config.n_embed) # position embeddings, outputs (B,T,C)\n",
        "        self.blocks = nn.Sequential(\n",
        "            *[TransformerBlock(config) for _ in range(config.n_blocks)]\n",
        "        ) # list of transformer blocks\n",
        "        self.layernorm = nn.LayerNorm(config.n_embed)\n",
        "        self.lm_head = nn.Linear(config.n_embed, config.vocab_size) # final linear layer, outputs (B,T,vocab_size)\n",
        "        self.tie_weights()\n",
        "\n",
        "    def tie_weights(self):\n",
        "      self.lm_head.weight = self.token_embedding_table.weight\n",
        "\n",
        "    # Takes input of shape (B,T) so B batches of T tokens (numbers)\n",
        "    def forward(self, tokens):\n",
        "        B, T = tokens.shape\n",
        "\n",
        "        # tokens and targets are both (B,T) tensor of integers\n",
        "        token_embeddings = self.token_embedding_table(tokens) # (B,T,C)\n",
        "        position_embeddings = self.position_embedding_table(torch.arange(T, device=self.config.device)) # (T,C)\n",
        "        x = token_embeddings + position_embeddings # broadcasting automatically turns position_embeddings into (B,T,C) by adding dim and repeating (T,C) B times\n",
        "\n",
        "        # apply transformer blocks\n",
        "        x = self.blocks(x)\n",
        "\n",
        "        # apply layernorm\n",
        "        x = self.layernorm(x)\n",
        "\n",
        "        # pass self attention into final layer to convert to vocab size dims\n",
        "        logits = self.lm_head(x) # (B,T,vocab_size)\n",
        "        return logits\n",
        "\n",
        "    def generate(self, tokens, max_new_tokens):\n",
        "        # tokens is (B, T) array of indices in the current context\n",
        "        for _ in range(max_new_tokens):\n",
        "            # crop tokens to make sure it doesn't exceed block_size\n",
        "            tokens_cropped = tokens[:, -self.config.block_size:]\n",
        "\n",
        "            # get the predictions\n",
        "            logits = self(tokens_cropped)\n",
        "            # focus only on the last time step\n",
        "            logits = logits[:, -1, :] # becomes (B, C)\n",
        "            # apply softmax to get probabilities\n",
        "            probs = F.softmax(logits, dim=-1) # (B, C)\n",
        "            # sample from the distribution\n",
        "            tokens_next = torch.multinomial(probs, num_samples=1) # (B, 1)\n",
        "            # append sampled index to the running sequence\n",
        "            tokens = torch.cat((tokens, tokens_next), dim=1) # (B, T+1)\n",
        "        return tokens"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RSwSvDvhDKu9"
      },
      "source": [
        "## Setup Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "IEHhmg0PDeEb"
      },
      "outputs": [],
      "source": [
        "!pip install -q torchinfo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "HaPAeUBAKY04"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "v55qhrEGvFk9"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.nn import functional as F\n",
        "import torchinfo\n",
        "\n",
        "# hyperparameters\n",
        "batch_size = 8\n",
        "block_size = 1024 # max context length\n",
        "learning_rate = 5e-5\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "eval_iters = 200\n",
        "n_embed = 768\n",
        "n_heads = 12\n",
        "n_blocks = 12\n",
        "dropout = 0.2\n",
        "vocab_size = 50_257 # tiktoken gpt2 tokenizer vocab size (tiktoken.encoding_for_model(\"gpt2\").n_vocab)\n",
        "# ------------\n",
        "\n",
        "config = GPTLanguageModelConfig(\n",
        "    block_size = block_size,\n",
        "    vocab_size = vocab_size,\n",
        "    n_embed = n_embed,\n",
        "    n_heads = n_heads,\n",
        "    n_blocks = n_blocks,\n",
        "    dropout_rate = dropout,\n",
        "    device = device\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eNU_jKKSoLTq",
        "outputId": "20247e22-b01b-45f9-a430-dc4618724032"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "===============================================================================================\n",
              "Layer (type:depth-idx)                        Output Shape              Param #\n",
              "===============================================================================================\n",
              "GPTLanguageModel                              [1, 1024, 50257]          --\n",
              "├─Embedding: 1-1                              [1, 1024, 768]            38,597,376\n",
              "├─Embedding: 1-2                              [1024, 768]               786,432\n",
              "├─Sequential: 1-3                             [1, 1024, 768]            --\n",
              "│    └─TransformerBlock: 2-1                  [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-1                    [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-2           [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-3                    [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-4                  [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-2                  [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-5                    [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-6           [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-7                    [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-8                  [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-3                  [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-9                    [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-10          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-11                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-12                 [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-4                  [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-13                   [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-14          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-15                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-16                 [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-5                  [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-17                   [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-18          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-19                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-20                 [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-6                  [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-21                   [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-22          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-23                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-24                 [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-7                  [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-25                   [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-26          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-27                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-28                 [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-8                  [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-29                   [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-30          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-31                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-32                 [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-9                  [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-33                   [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-34          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-35                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-36                 [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-10                 [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-37                   [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-38          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-39                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-40                 [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-11                 [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-41                   [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-42          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-43                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-44                 [1, 1024, 768]            4,722,432\n",
              "│    └─TransformerBlock: 2-12                 [1, 1024, 768]            --\n",
              "│    │    └─LayerNorm: 3-45                   [1, 1024, 768]            1,536\n",
              "│    │    └─MultiHeadAttention: 3-46          [1, 1024, 768]            2,360,064\n",
              "│    │    └─LayerNorm: 3-47                   [1, 1024, 768]            1,536\n",
              "│    │    └─FeedForward: 3-48                 [1, 1024, 768]            4,722,432\n",
              "├─LayerNorm: 1-4                              [1, 1024, 768]            1,536\n",
              "├─Linear: 1-5                                 [1, 1024, 50257]          38,647,633\n",
              "===============================================================================================\n",
              "Total params: 163,059,793\n",
              "Trainable params: 163,059,793\n",
              "Non-trainable params: 0\n",
              "Total mult-adds (M): 967.58\n",
              "===============================================================================================\n",
              "Input size (MB): 0.01\n",
              "Forward/backward pass size (MB): 1261.05\n",
              "Params size (MB): 652.24\n",
              "Estimated Total Size (MB): 1913.30\n",
              "==============================================================================================="
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model = GPTLanguageModel(config).to(device)\n",
        "torchinfo.summary(model, input_size=(1, config.block_size), dtypes=[torch.long])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zVy1GpxyW2A4",
        "outputId": "a4a37df3-9874-4f3a-ea68-b9bd8d08032d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "True\n"
          ]
        }
      ],
      "source": [
        "# NOTE: torchinfo doesn't recognize that final Linear layer shares weights with embedding layer so number of params is actually 124M\n",
        "print(model.lm_head.weight is model.token_embedding_table.weight)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wxzqvTXcVSVW"
      },
      "source": [
        "## Setup Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ha_KoLD3VUJ2",
        "outputId": "35a71ccb-dd69-4f9c-a520-83b4c66b8ba6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.0/542.0 kB\u001b[0m \u001b[31m34.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m401.2/401.2 kB\u001b[0m \u001b[31m33.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install -q tiktoken portalocker torchdata datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 588,
          "referenced_widgets": [
            "e74cf291ebd94e97b8fc0321a5d612bc",
            "fc0b56e88b02466980d15e49a46f675d",
            "945a9bd261d847e2a347bdd5ace4e262",
            "ad5db35ee56944a4a8921f532812a849",
            "2e11473576b341c8bd368da615079cec",
            "bc87b549597449df8f5bcd5e2ac0058b",
            "e8b0f11f0e054c7893b90369a9c78027",
            "5ad4801507084252a5ef4f2a9c0961fd",
            "a4c359dbd8154f50ac68829ef25c0077",
            "439e705dd0224f24ba4b87c832d63223",
            "83659c6054d049aa8a0ad6116cc81aa4",
            "54bd62de18a541e4ad975a357a0398d6",
            "acb0c22a1f1847c58d330c7edc5c2443",
            "a49377d74d8941dca985550539836b95",
            "759640724fd34ef89c6dab2c1f0e5c3b",
            "e1e1c5b4647945e3a9643f8913a07a02",
            "69b1edbdc1da4109b73ca441a56890ac",
            "9013a3e59a9241678c8368a7d97132a4",
            "f913baffa338406d8699c788777c6838",
            "4d27e238604849c58b55027f7a73de49",
            "ea27be20005748ff9989ee0b2afb42ba",
            "bf02744ba3d54a4ea5ad7b035675c126",
            "03e8d81eab884777aef78174ef78fe7c",
            "5aa38becd0284aafb618042f754c9feb",
            "acf5dd6f27ec4ceda9abe27a4d66e52b",
            "b72053c682d64d00b8ff21cc202f2200",
            "5c9201742a264778b3e369eda5e9b7c7",
            "901b168e5fd740879b79ff42efd5939f",
            "b5817cc8fc694a7ebccddb959e7db3b2",
            "455dbbde3a794cfdb0031c959f3c06c9",
            "2be1ad07fa2e4c5cad4d2fa60897c127",
            "2a54afe25e034e01baa8ad6425880b97",
            "0b33eec826664941a37cf1dc579b96e3",
            "fc96c976349149c0a268af89a389e981",
            "9865796f7e794df09f0d1b96e86f6376",
            "a278651cd19a49f9ab938e5a558c7dd7",
            "145434eb8a3a47f78776242c5eeaf900",
            "a9941b5b087c47dca600141da2227bc8",
            "1249bd1335384f6e8b9feb7564fe037f",
            "0415cf3d2acc4f579d58e8752c0ff783",
            "0525522cf3674ea59efaff41e81c20b1",
            "c431c4f6daf94737a15776d49cc111e9",
            "3e2651a570484074a4591dc2d7f39624",
            "59dfac73d1b54a15a1272dc9e65b678f",
            "bf2c9c063709484f8f7ab0acb3fcc187",
            "2b8dca4e41ae4ad1a9c3b7933ac86204",
            "15bdab9ac0b044e186a82e132b959fa3",
            "6a14e21659164ca5b33e3e293817331f",
            "5fe591be7a394c1a8677ff97bd31ad78",
            "4e12725dc6274c9485da34a96bad1d06",
            "8cc2e2b3a79f49e596ab42145a865d81",
            "a60994c478914c2ea991e5b8d5fa0fcc",
            "d498718e04834baeba40722a22009720",
            "5858c62bb043405eb85c0b6f75ad9bee",
            "5ec8c570840f4460a76fe55b48e880ef",
            "d4014bcba25e4616bdc24b211829485e",
            "2cbf30fc05274ee8bee3ce4bd9a192cb",
            "1b0cf65c895d4ecb9ff4c65e788b69d8",
            "7998ab41f629451fabf4099c5cb5ae74",
            "4f9054d7b9844e6c91d0c91c100d6b4d",
            "4306fb54536b4eaa933a06cdfdda28c7",
            "3414426d950b4179b7b0aedd1f13526c",
            "c62f3713715c4927830f6ad84eb9e389",
            "957640025d114a869ac44dff151a33b7",
            "cfe06194edda452d841154aab3c68479",
            "0fd3c9b568d44a64888768d6ad0f917a",
            "c97d33b9780849f2ba1cab280cc5319c",
            "2d5406cf96954ef0ad4aee738f5dafcb",
            "3b78228f61a0482185e6ab90ab214a13",
            "3dfebdadaf834f578bba664f405f251c",
            "6f7ad7ad8779487ab58b7ee4f18e4718",
            "80820456829c441fa12de7c850ee79e9",
            "439dbb74e7de45b385d76d3b4af7c7d6",
            "e416a38ecab94440990ca49d9a9f13db",
            "d2c57c14fc474690aacdc6bd4b15e11f",
            "668fc371deff40e2b0a9165105c77dfc",
            "1f0c4e681c904f3eb2503111e413f902"
          ]
        },
        "id": "ntER4J7tPx6Q",
        "outputId": "69ea24c0-790c-4209-9e79-f558b5a0180a"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e74cf291ebd94e97b8fc0321a5d612bc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/10.5k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "54bd62de18a541e4ad975a357a0398d6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading data:   0%|          | 0.00/733k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "03e8d81eab884777aef78174ef78fe7c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading data:   0%|          | 0.00/6.36M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fc96c976349149c0a268af89a389e981",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading data:   0%|          | 0.00/657k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bf2c9c063709484f8f7ab0acb3fcc187",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating test split:   0%|          | 0/4358 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d4014bcba25e4616bdc24b211829485e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating train split:   0%|          | 0/36718 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c97d33b9780849f2ba1cab280cc5319c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating validation split:   0%|          | 0/3760 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    test: Dataset({\n",
              "        features: ['text'],\n",
              "        num_rows: 4358\n",
              "    })\n",
              "    train: Dataset({\n",
              "        features: ['text'],\n",
              "        num_rows: 36718\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['text'],\n",
              "        num_rows: 3760\n",
              "    })\n",
              "})"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "os.environ[\"HF_TOKEN\"] = \"<INSERT HUGGING FACE TOKEN>\"\n",
        "dataset = load_dataset('wikitext', 'wikitext-2-raw-v1')\n",
        "dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "Rg9uggo75Rxr"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import Dataset\n",
        "import tiktoken\n",
        "\n",
        "class WikitextDataset(Dataset):\n",
        "    def __init__(self, dataset, block_size):\n",
        "      super().__init__()\n",
        "      self.block_size = block_size\n",
        "      self.tokenizer = tiktoken.encoding_for_model(\"gpt2\")\n",
        "\n",
        "      data = [text for text in dataset['text'] if text != \"\"] # remove empty samples\n",
        "      data = \" \".join(data) # combine all samples into single string\n",
        "      self.data = self.tokenizer.encode(data) # tokenize items in string\n",
        "\n",
        "    def __len__(self):\n",
        "      return ((len(self.data) - self.block_size) // self.block_size)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "      start_idx = idx * self.block_size\n",
        "      x = self.data[start_idx:start_idx+self.block_size]\n",
        "      y = self.data[start_idx+1:start_idx+self.block_size+1]\n",
        "      return torch.tensor(x, dtype=torch.long), torch.tensor(y, dtype=torch.long)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "YxNOLc_akjf1"
      },
      "outputs": [],
      "source": [
        "train_dataset = WikitextDataset(dataset['train'], block_size=block_size)\n",
        "train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size)\n",
        "\n",
        "val_dataset = WikitextDataset(dataset['validation'], block_size=block_size)\n",
        "val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mVe6MsSPpYF_",
        "outputId": "e22adc36-d5f2-4e54-e4d8-d0a850b092d8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(2358, 242)"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(train_dataset), len(val_dataset)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ttOINaTMrFGs",
        "outputId": "a3cc2c82-5910-4085-bfff-37bca5c480ab"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(295, 31)"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(train_dataloader), len(val_dataloader)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0zyrUuzN2Wix"
      },
      "source": [
        "## Training Code"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "AkTVTdKq4owh"
      },
      "outputs": [],
      "source": [
        "def accuracy_fn(y_true, y_pred):\n",
        "    \"\"\"Calculates accuracy between truth labels and predictions.\n",
        "\n",
        "    Args:\n",
        "        y_true (torch.Tensor): Truth labels for predictions.\n",
        "        y_pred (torch.Tensor): Predictions to be compared to predictions.\n",
        "\n",
        "    Returns:\n",
        "        [torch.float]: Accuracy value between y_true and y_pred, e.g. 78.45\n",
        "    \"\"\"\n",
        "    correct = torch.eq(y_true, y_pred).sum().item()\n",
        "    acc = (correct / len(y_pred)) * 100\n",
        "    return acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "YQK3cWzu3YCc"
      },
      "outputs": [],
      "source": [
        "def train_step(\n",
        "    model: nn.Module,\n",
        "    loss_fn: nn.Module,\n",
        "    optimizer: torch.optim.Optimizer,\n",
        "    data_loader: torch.utils.data.DataLoader,\n",
        "    accuracy_fn,\n",
        "    device: torch.device):\n",
        "  model.train()\n",
        "\n",
        "  train_acc, train_loss = 0, 0\n",
        "\n",
        "  # train_dataloader iters (x, y), enumerate adds iter number (batch num)\n",
        "  for batch_num, (X, y) in enumerate(data_loader):\n",
        "    try:\n",
        "      X, y = X.to(device), y.to(device)\n",
        "\n",
        "      logits = model(X)\n",
        "      B, T, C = logits.shape\n",
        "\n",
        "      y_pred = logits.view(B*T, C)\n",
        "      y = y.view(B*T)\n",
        "\n",
        "      # add loss for every batch\n",
        "      loss = loss_fn(y_pred, y)\n",
        "      train_loss += loss\n",
        "      train_acc += accuracy_fn(y_true=y, y_pred=y_pred.argmax(dim=-1))\n",
        "\n",
        "      optimizer.zero_grad()\n",
        "\n",
        "      loss.backward()\n",
        "\n",
        "      optimizer.step()\n",
        "\n",
        "      if batch_num % 50 == 0:\n",
        "        print(f\"{batch_num * len(X)}/{len(data_loader.dataset)} samples...\")\n",
        "        curr_acc = train_acc / (batch_num + 1)\n",
        "        curr_loss = train_loss / (batch_num + 1)\n",
        "        print(f\"Batch number {batch_num}. Train loss: {curr_loss:.4f} | Train acc: {curr_acc:.4f}%\")\n",
        "    except Exception as e:\n",
        "        torch.set_printoptions(threshold=float('inf'))\n",
        "        print(\"Exception:\", e)\n",
        "        print(\"X:\", X)\n",
        "        print(\"y:\", y)\n",
        "        raise e\n",
        "\n",
        "  # get average loss per batch?\n",
        "  train_loss /= len(data_loader)\n",
        "  train_acc /= len(data_loader)\n",
        "\n",
        "  print(f\"\\nEnd of epoch. Train loss: {train_loss:.4f} | Train acc: {train_acc:.4f}%\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "6S5adz8X3nB4"
      },
      "outputs": [],
      "source": [
        "def val_step(model: torch.nn.Module,\n",
        "              loss_fn: torch.nn.Module,\n",
        "              data_loader: torch.utils.data.DataLoader,\n",
        "              accuracy_fn,\n",
        "              device: torch.device = device):\n",
        "  \"\"\"Performs a validation loop step on model going over data_loader.\"\"\"\n",
        "  val_loss, val_acc = 0, 0\n",
        "\n",
        "  # Put the model in eval mode\n",
        "  model.eval()\n",
        "\n",
        "  # Turn on inference mode context manager\n",
        "  with torch.inference_mode():\n",
        "    for X, y in data_loader:\n",
        "      # Send the data to the target device\n",
        "      X, y = X.to(device), y.to(device)\n",
        "\n",
        "      # 1. Forward pass (outputs raw logits)\n",
        "      logits = model(X)\n",
        "      B, T, C = logits.shape\n",
        "\n",
        "      val_pred = logits.view(B*T, C)\n",
        "      y = y.view(B*T)\n",
        "\n",
        "      # 2. Calculuate the loss/acc\n",
        "      val_loss += loss_fn(val_pred, y)\n",
        "      val_acc += accuracy_fn(y_true=y,\n",
        "                              y_pred=val_pred.argmax(dim=-1)) # go from logits -> prediction labels\n",
        "\n",
        "    # Adjust metrics and print out\n",
        "    val_loss /= len(data_loader)\n",
        "    val_acc /= len(data_loader)\n",
        "    print(f\"Validation loss: {val_loss:.5f} | Validation acc: {val_acc:.2f}%\\n\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JbakM5DC4Lwh"
      },
      "source": [
        "## Train Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "m02I-Wtnaz0C"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5xaUzBzv4NI8"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n",
        "os.environ[\"TORCH_USE_CUDA_DSA\"] = \"1\"\n",
        "\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "4194368d37304ae9a047cf31eaf0be62",
            "dcfd88d4584a4e3997cf26cb28715525",
            "773235daa69046898a02c1364e37c08e",
            "4266b3a7bbfa493d82bbd25d3334e893",
            "293295be247c4312ad52af956a514080",
            "7d31ec9e163547e78fb70cd9f4639d60",
            "9e471f3ae76d4d27bed6f25c12055046",
            "63a630710aa6466fb361b190f5974961",
            "0f86492fca264fab98dd4fefeb9aa7bb",
            "f8fe859b5ea44c158cfc204ad1c6174b",
            "f4ae2374450e4c7fa2884c4ca6ea942f"
          ]
        },
        "id": "TmLXi0J7DSNB",
        "outputId": "b6f11d3d-4a51-4ace-e9a8-4291aa5aed26"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4194368d37304ae9a047cf31eaf0be62",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/9 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch: 0\n",
            "------\n",
            "0/2358 samples...\n",
            "Batch number 0. Train loss: 12.2595 | Train acc: 6.4575%\n",
            "400/2358 samples...\n",
            "Batch number 50. Train loss: 11.6902 | Train acc: 7.9944%\n",
            "800/2358 samples...\n",
            "Batch number 100. Train loss: 11.4962 | Train acc: 7.9968%\n",
            "1200/2358 samples...\n",
            "Batch number 150. Train loss: 11.3587 | Train acc: 7.9857%\n",
            "1600/2358 samples...\n",
            "Batch number 200. Train loss: 11.2605 | Train acc: 8.0313%\n",
            "2000/2358 samples...\n",
            "Batch number 250. Train loss: 11.1831 | Train acc: 7.9886%\n",
            "\n",
            "End of epoch. Train loss: 11.1318 | Train acc: 8.0008%\n",
            "Validation loss: 8.43278 | Validation acc: 14.33%\n",
            "\n",
            "Epoch: 1\n",
            "------\n",
            "0/2358 samples...\n",
            "Batch number 0. Train loss: 11.4582 | Train acc: 7.8735%\n",
            "400/2358 samples...\n",
            "Batch number 50. Train loss: 11.2236 | Train acc: 8.2843%\n",
            "800/2358 samples...\n",
            "Batch number 100. Train loss: 11.0885 | Train acc: 8.2671%\n",
            "1200/2358 samples...\n",
            "Batch number 150. Train loss: 10.9897 | Train acc: 8.2412%\n",
            "1600/2358 samples...\n",
            "Batch number 200. Train loss: 10.9178 | Train acc: 8.2701%\n",
            "2000/2358 samples...\n",
            "Batch number 250. Train loss: 10.8628 | Train acc: 8.2182%\n",
            "\n",
            "End of epoch. Train loss: 10.8265 | Train acc: 8.2312%\n",
            "Validation loss: 8.26479 | Validation acc: 14.57%\n",
            "\n",
            "Epoch: 2\n",
            "------\n",
            "0/2358 samples...\n",
            "Batch number 0. Train loss: 11.1831 | Train acc: 7.7881%\n",
            "400/2358 samples...\n",
            "Batch number 50. Train loss: 10.9900 | Train acc: 8.3862%\n",
            "800/2358 samples...\n",
            "Batch number 100. Train loss: 10.8660 | Train acc: 8.4010%\n",
            "1200/2358 samples...\n",
            "Batch number 150. Train loss: 10.7725 | Train acc: 8.4031%\n",
            "1600/2358 samples...\n",
            "Batch number 200. Train loss: 10.7066 | Train acc: 8.4352%\n",
            "2000/2358 samples...\n",
            "Batch number 250. Train loss: 10.6543 | Train acc: 8.3859%\n",
            "\n",
            "End of epoch. Train loss: 10.6222 | Train acc: 8.3986%\n",
            "Validation loss: 8.13993 | Validation acc: 14.82%\n",
            "\n",
            "Epoch: 3\n",
            "------\n",
            "0/2358 samples...\n",
            "Batch number 0. Train loss: 10.9530 | Train acc: 7.8125%\n",
            "400/2358 samples...\n",
            "Batch number 50. Train loss: 10.7970 | Train acc: 8.6371%\n",
            "800/2358 samples...\n",
            "Batch number 100. Train loss: 10.6777 | Train acc: 8.6483%\n",
            "1200/2358 samples...\n",
            "Batch number 150. Train loss: 10.5896 | Train acc: 8.6123%\n",
            "1600/2358 samples...\n",
            "Batch number 200. Train loss: 10.5264 | Train acc: 8.6235%\n",
            "2000/2358 samples...\n",
            "Batch number 250. Train loss: 10.4789 | Train acc: 8.5711%\n",
            "\n",
            "End of epoch. Train loss: 10.4478 | Train acc: 8.5918%\n",
            "Validation loss: 8.03656 | Validation acc: 14.97%\n",
            "\n",
            "Epoch: 4\n",
            "------\n",
            "0/2358 samples...\n",
            "Batch number 0. Train loss: 10.7635 | Train acc: 7.9956%\n",
            "400/2358 samples...\n",
            "Batch number 50. Train loss: 10.6102 | Train acc: 8.8336%\n",
            "800/2358 samples...\n",
            "Batch number 100. Train loss: 10.4922 | Train acc: 8.8456%\n",
            "1200/2358 samples...\n",
            "Batch number 150. Train loss: 10.4069 | Train acc: 8.8227%\n",
            "1600/2358 samples...\n",
            "Batch number 200. Train loss: 10.3502 | Train acc: 8.8632%\n",
            "2000/2358 samples...\n",
            "Batch number 250. Train loss: 10.3065 | Train acc: 8.8083%\n",
            "\n",
            "End of epoch. Train loss: 10.2786 | Train acc: 8.8178%\n",
            "Validation loss: 7.98934 | Validation acc: 15.04%\n",
            "\n",
            "Epoch: 5\n",
            "------\n",
            "0/2358 samples...\n",
            "Batch number 0. Train loss: 10.6226 | Train acc: 8.7402%\n",
            "400/2358 samples...\n",
            "Batch number 50. Train loss: 10.4268 | Train acc: 9.0586%\n",
            "800/2358 samples...\n",
            "Batch number 100. Train loss: 10.3191 | Train acc: 9.0693%\n",
            "1200/2358 samples...\n",
            "Batch number 150. Train loss: 10.2380 | Train acc: 9.0583%\n",
            "1600/2358 samples...\n",
            "Batch number 200. Train loss: 10.1850 | Train acc: 9.0841%\n",
            "2000/2358 samples...\n",
            "Batch number 250. Train loss: 10.1459 | Train acc: 9.0209%\n",
            "\n",
            "End of epoch. Train loss: 10.1195 | Train acc: 9.0450%\n",
            "Validation loss: 7.87649 | Validation acc: 15.23%\n",
            "\n",
            "Epoch: 6\n",
            "------\n",
            "0/2358 samples...\n",
            "Batch number 0. Train loss: 10.4418 | Train acc: 8.3618%\n",
            "400/2358 samples...\n",
            "Batch number 50. Train loss: 10.2526 | Train acc: 9.2807%\n",
            "800/2358 samples...\n",
            "Batch number 100. Train loss: 10.1476 | Train acc: 9.2789%\n",
            "1200/2358 samples...\n",
            "Batch number 150. Train loss: 10.0754 | Train acc: 9.2486%\n",
            "1600/2358 samples...\n",
            "Batch number 200. Train loss: 10.0261 | Train acc: 9.2657%\n",
            "2000/2358 samples...\n",
            "Batch number 250. Train loss: 9.9921 | Train acc: 9.2119%\n",
            "\n",
            "End of epoch. Train loss: 9.9682 | Train acc: 9.2424%\n",
            "Validation loss: 7.80862 | Validation acc: 15.28%\n",
            "\n",
            "Epoch: 7\n",
            "------\n",
            "0/2358 samples...\n",
            "Batch number 0. Train loss: 10.3037 | Train acc: 8.1909%\n",
            "400/2358 samples...\n",
            "Batch number 50. Train loss: 10.1008 | Train acc: 9.4683%\n",
            "800/2358 samples...\n",
            "Batch number 100. Train loss: 10.0055 | Train acc: 9.4933%\n",
            "1200/2358 samples...\n",
            "Batch number 150. Train loss: 9.9334 | Train acc: 9.4496%\n",
            "1600/2358 samples...\n",
            "Batch number 200. Train loss: 9.8844 | Train acc: 9.4797%\n",
            "2000/2358 samples...\n",
            "Batch number 250. Train loss: 9.8515 | Train acc: 9.4231%\n",
            "\n",
            "End of epoch. Train loss: 9.8286 | Train acc: 9.4507%\n",
            "Validation loss: 7.78718 | Validation acc: 15.38%\n",
            "\n",
            "Epoch: 8\n",
            "------\n",
            "0/2358 samples...\n",
            "Batch number 0. Train loss: 10.1362 | Train acc: 8.1299%\n",
            "400/2358 samples...\n",
            "Batch number 50. Train loss: 9.9518 | Train acc: 9.6294%\n",
            "800/2358 samples...\n",
            "Batch number 100. Train loss: 9.8581 | Train acc: 9.6525%\n",
            "1200/2358 samples...\n",
            "Batch number 150. Train loss: 9.7935 | Train acc: 9.6136%\n",
            "1600/2358 samples...\n",
            "Batch number 200. Train loss: 9.7502 | Train acc: 9.6372%\n",
            "2000/2358 samples...\n",
            "Batch number 250. Train loss: 9.7205 | Train acc: 9.5912%\n",
            "\n",
            "End of epoch. Train loss: 9.7001 | Train acc: 9.6199%\n",
            "Validation loss: 7.76050 | Validation acc: 15.33%\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from tqdm.auto import tqdm\n",
        "\n",
        "epochs = 9\n",
        "for epoch in tqdm(range(epochs)):\n",
        "    print(f\"Epoch: {epoch}\\n------\")\n",
        "\n",
        "    train_step(model=model,\n",
        "              loss_fn=loss_fn,\n",
        "              optimizer=optimizer,\n",
        "              data_loader=train_dataloader,\n",
        "              accuracy_fn=accuracy_fn,\n",
        "              device=device)\n",
        "    val_step(model=model,\n",
        "              data_loader=val_dataloader,\n",
        "              loss_fn=loss_fn,\n",
        "              accuracy_fn=accuracy_fn)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lY3XflsKqns5"
      },
      "source": [
        "## Save Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CtrRoMvcdO3-",
        "outputId": "197510f2-d803-4279-d7e6-f1e4490c3804"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "import shutil\n",
        "import os\n",
        "\n",
        "# Mount Google Drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# The destination directory in Google Drive\n",
        "BASE_DIR = '/content/drive/My Drive/Learning/gpt-2-small'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WXvMUrvPCGh_"
      },
      "outputs": [],
      "source": [
        "torch.save(model.state_dict(), f\"{BASE_DIR}/model-tied-10-epochs.pth\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IbEg3e90q4jB"
      },
      "source": [
        "## Load Model and Generate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "OVQUQCYnkC5X",
        "outputId": "0eababd8-eaac-4740-fe32-9a88d7643276"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model = GPTLanguageModel(config).to(device)\n",
        "model.load_state_dict(torch.load(f\"{BASE_DIR}/model-tied-10-epochs.pth\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qby9pmMpi6Qd",
        "outputId": "0a339aca-d930-4b9e-94dd-1d39a7c8ab95"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "! Dou theatre in for thecor in religious thefer fate . \n",
            "  Pl , the side of the fountain different from the nude to the 2008 ( 3 to the . Falls . \n",
            "  Thes is a image . \n",
            "  = \n",
            " \n",
            " \n",
            "  Or Lust , a sidelines , and the three several Mother , andbacks mol a Empire of the the design enlightenment began , the .  = = = \n",
            "  natural the salt @-@ 000 the that was the called Blue , but the ranked in the = = = = \n",
            "  = \n",
            " \n",
            " \n",
            "  In the Gardner and can be a receives dome , and used , inalions to the Katie \" theops and earlyung \" , United States , Premier V des of theai , the Trafford , the instance released , the M Bush , vegetation takes , and humans and the wid oneren . \n",
            " \n",
            " \n",
            "  = = = = = \n",
            " \n",
            "  Theint to be in the people its nation its in the the first 150 he was 10 @-@ rice of the the provides lower @-@ Puerto � is and a off all in a originally was tongue of the South most sponsored , which 11 11 to theures , the largest Cory\n"
          ]
        }
      ],
      "source": [
        "import tiktoken\n",
        "\n",
        "encoder = tiktoken.encoding_for_model(\"gpt2\")\n",
        "context = torch.zeros((1, 1), dtype=torch.long, device=device)\n",
        "\n",
        "with torch.inference_mode():\n",
        "  print(encoder.decode(model.generate(context, max_new_tokens=256)[0].tolist()))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "03e8d81eab884777aef78174ef78fe7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5aa38becd0284aafb618042f754c9feb",
              "IPY_MODEL_acf5dd6f27ec4ceda9abe27a4d66e52b",
              "IPY_MODEL_b72053c682d64d00b8ff21cc202f2200"
            ],
            "layout": "IPY_MODEL_5c9201742a264778b3e369eda5e9b7c7"
          }
        },
        "0415cf3d2acc4f579d58e8752c0ff783": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0525522cf3674ea59efaff41e81c20b1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b33eec826664941a37cf1dc579b96e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f86492fca264fab98dd4fefeb9aa7bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0fd3c9b568d44a64888768d6ad0f917a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1249bd1335384f6e8b9feb7564fe037f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "145434eb8a3a47f78776242c5eeaf900": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e2651a570484074a4591dc2d7f39624",
            "placeholder": "​",
            "style": "IPY_MODEL_59dfac73d1b54a15a1272dc9e65b678f",
            "value": " 657k/657k [00:00&lt;00:00, 12.9MB/s]"
          }
        },
        "15bdab9ac0b044e186a82e132b959fa3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a60994c478914c2ea991e5b8d5fa0fcc",
            "max": 4358,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d498718e04834baeba40722a22009720",
            "value": 4358
          }
        },
        "1b0cf65c895d4ecb9ff4c65e788b69d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c62f3713715c4927830f6ad84eb9e389",
            "max": 36718,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_957640025d114a869ac44dff151a33b7",
            "value": 36718
          }
        },
        "1f0c4e681c904f3eb2503111e413f902": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "293295be247c4312ad52af956a514080": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a54afe25e034e01baa8ad6425880b97": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b8dca4e41ae4ad1a9c3b7933ac86204": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4e12725dc6274c9485da34a96bad1d06",
            "placeholder": "​",
            "style": "IPY_MODEL_8cc2e2b3a79f49e596ab42145a865d81",
            "value": "Generating test split: 100%"
          }
        },
        "2be1ad07fa2e4c5cad4d2fa60897c127": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2cbf30fc05274ee8bee3ce4bd9a192cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4306fb54536b4eaa933a06cdfdda28c7",
            "placeholder": "​",
            "style": "IPY_MODEL_3414426d950b4179b7b0aedd1f13526c",
            "value": "Generating train split: 100%"
          }
        },
        "2d5406cf96954ef0ad4aee738f5dafcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_80820456829c441fa12de7c850ee79e9",
            "placeholder": "​",
            "style": "IPY_MODEL_439dbb74e7de45b385d76d3b4af7c7d6",
            "value": "Generating validation split: 100%"
          }
        },
        "2e11473576b341c8bd368da615079cec": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3414426d950b4179b7b0aedd1f13526c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3b78228f61a0482185e6ab90ab214a13": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e416a38ecab94440990ca49d9a9f13db",
            "max": 3760,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d2c57c14fc474690aacdc6bd4b15e11f",
            "value": 3760
          }
        },
        "3dfebdadaf834f578bba664f405f251c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_668fc371deff40e2b0a9165105c77dfc",
            "placeholder": "​",
            "style": "IPY_MODEL_1f0c4e681c904f3eb2503111e413f902",
            "value": " 3760/3760 [00:00&lt;00:00, 101513.85 examples/s]"
          }
        },
        "3e2651a570484074a4591dc2d7f39624": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4194368d37304ae9a047cf31eaf0be62": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dcfd88d4584a4e3997cf26cb28715525",
              "IPY_MODEL_773235daa69046898a02c1364e37c08e",
              "IPY_MODEL_4266b3a7bbfa493d82bbd25d3334e893"
            ],
            "layout": "IPY_MODEL_293295be247c4312ad52af956a514080"
          }
        },
        "4266b3a7bbfa493d82bbd25d3334e893": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f8fe859b5ea44c158cfc204ad1c6174b",
            "placeholder": "​",
            "style": "IPY_MODEL_f4ae2374450e4c7fa2884c4ca6ea942f",
            "value": " 9/9 [40:17&lt;00:00, 268.54s/it]"
          }
        },
        "4306fb54536b4eaa933a06cdfdda28c7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "439dbb74e7de45b385d76d3b4af7c7d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "439e705dd0224f24ba4b87c832d63223": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "455dbbde3a794cfdb0031c959f3c06c9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d27e238604849c58b55027f7a73de49": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4e12725dc6274c9485da34a96bad1d06": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4f9054d7b9844e6c91d0c91c100d6b4d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54bd62de18a541e4ad975a357a0398d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_acb0c22a1f1847c58d330c7edc5c2443",
              "IPY_MODEL_a49377d74d8941dca985550539836b95",
              "IPY_MODEL_759640724fd34ef89c6dab2c1f0e5c3b"
            ],
            "layout": "IPY_MODEL_e1e1c5b4647945e3a9643f8913a07a02"
          }
        },
        "5858c62bb043405eb85c0b6f75ad9bee": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "59dfac73d1b54a15a1272dc9e65b678f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5aa38becd0284aafb618042f754c9feb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_901b168e5fd740879b79ff42efd5939f",
            "placeholder": "​",
            "style": "IPY_MODEL_b5817cc8fc694a7ebccddb959e7db3b2",
            "value": "Downloading data: 100%"
          }
        },
        "5ad4801507084252a5ef4f2a9c0961fd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c9201742a264778b3e369eda5e9b7c7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ec8c570840f4460a76fe55b48e880ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5fe591be7a394c1a8677ff97bd31ad78": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "63a630710aa6466fb361b190f5974961": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "668fc371deff40e2b0a9165105c77dfc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69b1edbdc1da4109b73ca441a56890ac": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a14e21659164ca5b33e3e293817331f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5858c62bb043405eb85c0b6f75ad9bee",
            "placeholder": "​",
            "style": "IPY_MODEL_5ec8c570840f4460a76fe55b48e880ef",
            "value": " 4358/4358 [00:00&lt;00:00, 64509.31 examples/s]"
          }
        },
        "6f7ad7ad8779487ab58b7ee4f18e4718": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "759640724fd34ef89c6dab2c1f0e5c3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ea27be20005748ff9989ee0b2afb42ba",
            "placeholder": "​",
            "style": "IPY_MODEL_bf02744ba3d54a4ea5ad7b035675c126",
            "value": " 733k/733k [00:00&lt;00:00, 4.85MB/s]"
          }
        },
        "773235daa69046898a02c1364e37c08e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_63a630710aa6466fb361b190f5974961",
            "max": 9,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0f86492fca264fab98dd4fefeb9aa7bb",
            "value": 9
          }
        },
        "7998ab41f629451fabf4099c5cb5ae74": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cfe06194edda452d841154aab3c68479",
            "placeholder": "​",
            "style": "IPY_MODEL_0fd3c9b568d44a64888768d6ad0f917a",
            "value": " 36718/36718 [00:00&lt;00:00, 419374.43 examples/s]"
          }
        },
        "7d31ec9e163547e78fb70cd9f4639d60": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80820456829c441fa12de7c850ee79e9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83659c6054d049aa8a0ad6116cc81aa4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8cc2e2b3a79f49e596ab42145a865d81": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9013a3e59a9241678c8368a7d97132a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "901b168e5fd740879b79ff42efd5939f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "945a9bd261d847e2a347bdd5ace4e262": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ad4801507084252a5ef4f2a9c0961fd",
            "max": 10464,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a4c359dbd8154f50ac68829ef25c0077",
            "value": 10464
          }
        },
        "957640025d114a869ac44dff151a33b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9865796f7e794df09f0d1b96e86f6376": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1249bd1335384f6e8b9feb7564fe037f",
            "placeholder": "​",
            "style": "IPY_MODEL_0415cf3d2acc4f579d58e8752c0ff783",
            "value": "Downloading data: 100%"
          }
        },
        "9e471f3ae76d4d27bed6f25c12055046": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a278651cd19a49f9ab938e5a558c7dd7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0525522cf3674ea59efaff41e81c20b1",
            "max": 657209,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c431c4f6daf94737a15776d49cc111e9",
            "value": 657209
          }
        },
        "a49377d74d8941dca985550539836b95": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f913baffa338406d8699c788777c6838",
            "max": 732610,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4d27e238604849c58b55027f7a73de49",
            "value": 732610
          }
        },
        "a4c359dbd8154f50ac68829ef25c0077": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a60994c478914c2ea991e5b8d5fa0fcc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9941b5b087c47dca600141da2227bc8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acb0c22a1f1847c58d330c7edc5c2443": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69b1edbdc1da4109b73ca441a56890ac",
            "placeholder": "​",
            "style": "IPY_MODEL_9013a3e59a9241678c8368a7d97132a4",
            "value": "Downloading data: 100%"
          }
        },
        "acf5dd6f27ec4ceda9abe27a4d66e52b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_455dbbde3a794cfdb0031c959f3c06c9",
            "max": 6357543,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2be1ad07fa2e4c5cad4d2fa60897c127",
            "value": 6357543
          }
        },
        "ad5db35ee56944a4a8921f532812a849": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_439e705dd0224f24ba4b87c832d63223",
            "placeholder": "​",
            "style": "IPY_MODEL_83659c6054d049aa8a0ad6116cc81aa4",
            "value": " 10.5k/10.5k [00:00&lt;00:00, 623kB/s]"
          }
        },
        "b5817cc8fc694a7ebccddb959e7db3b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b72053c682d64d00b8ff21cc202f2200": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a54afe25e034e01baa8ad6425880b97",
            "placeholder": "​",
            "style": "IPY_MODEL_0b33eec826664941a37cf1dc579b96e3",
            "value": " 6.36M/6.36M [00:00&lt;00:00, 62.8MB/s]"
          }
        },
        "bc87b549597449df8f5bcd5e2ac0058b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf02744ba3d54a4ea5ad7b035675c126": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bf2c9c063709484f8f7ab0acb3fcc187": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2b8dca4e41ae4ad1a9c3b7933ac86204",
              "IPY_MODEL_15bdab9ac0b044e186a82e132b959fa3",
              "IPY_MODEL_6a14e21659164ca5b33e3e293817331f"
            ],
            "layout": "IPY_MODEL_5fe591be7a394c1a8677ff97bd31ad78"
          }
        },
        "c431c4f6daf94737a15776d49cc111e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c62f3713715c4927830f6ad84eb9e389": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c97d33b9780849f2ba1cab280cc5319c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2d5406cf96954ef0ad4aee738f5dafcb",
              "IPY_MODEL_3b78228f61a0482185e6ab90ab214a13",
              "IPY_MODEL_3dfebdadaf834f578bba664f405f251c"
            ],
            "layout": "IPY_MODEL_6f7ad7ad8779487ab58b7ee4f18e4718"
          }
        },
        "cfe06194edda452d841154aab3c68479": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d2c57c14fc474690aacdc6bd4b15e11f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d4014bcba25e4616bdc24b211829485e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2cbf30fc05274ee8bee3ce4bd9a192cb",
              "IPY_MODEL_1b0cf65c895d4ecb9ff4c65e788b69d8",
              "IPY_MODEL_7998ab41f629451fabf4099c5cb5ae74"
            ],
            "layout": "IPY_MODEL_4f9054d7b9844e6c91d0c91c100d6b4d"
          }
        },
        "d498718e04834baeba40722a22009720": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dcfd88d4584a4e3997cf26cb28715525": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7d31ec9e163547e78fb70cd9f4639d60",
            "placeholder": "​",
            "style": "IPY_MODEL_9e471f3ae76d4d27bed6f25c12055046",
            "value": "100%"
          }
        },
        "e1e1c5b4647945e3a9643f8913a07a02": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e416a38ecab94440990ca49d9a9f13db": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e74cf291ebd94e97b8fc0321a5d612bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fc0b56e88b02466980d15e49a46f675d",
              "IPY_MODEL_945a9bd261d847e2a347bdd5ace4e262",
              "IPY_MODEL_ad5db35ee56944a4a8921f532812a849"
            ],
            "layout": "IPY_MODEL_2e11473576b341c8bd368da615079cec"
          }
        },
        "e8b0f11f0e054c7893b90369a9c78027": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ea27be20005748ff9989ee0b2afb42ba": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4ae2374450e4c7fa2884c4ca6ea942f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f8fe859b5ea44c158cfc204ad1c6174b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f913baffa338406d8699c788777c6838": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc0b56e88b02466980d15e49a46f675d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc87b549597449df8f5bcd5e2ac0058b",
            "placeholder": "​",
            "style": "IPY_MODEL_e8b0f11f0e054c7893b90369a9c78027",
            "value": "Downloading readme: 100%"
          }
        },
        "fc96c976349149c0a268af89a389e981": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9865796f7e794df09f0d1b96e86f6376",
              "IPY_MODEL_a278651cd19a49f9ab938e5a558c7dd7",
              "IPY_MODEL_145434eb8a3a47f78776242c5eeaf900"
            ],
            "layout": "IPY_MODEL_a9941b5b087c47dca600141da2227bc8"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}