variable_length.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from transformer_lens import HookedTransformer, HookedTransformerConfig, utils\n",
    "import circuitsvis as cv\n",
    "import plotly.express as px\n",
    "import pandas as pd\n",
    "from fancy_einsum import einsum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model parameter constants\n",
    "N_LAYERS = 1\n",
    "N_HEADS = 1\n",
    "D_MODEL = 32\n",
    "D_HEAD = 32\n",
    "D_MLP = None\n",
    "D_VOCAB = 64\n",
    "SEED = 123\n",
    "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def line(tensor, line_labels=None, yaxis=\"\", xaxis=\"\", **kwargs):\n",
    "    if isinstance(tensor, torch.Tensor):\n",
    "        tensor = tensor.detach().numpy()\n",
    "    \n",
    "    df = pd.DataFrame({\n",
    "        xaxis: np.arange(len(tensor)),\n",
    "        yaxis: tensor\n",
    "    })\n",
    "    \n",
    "    fig = px.line(df, x=xaxis, y=yaxis, labels={xaxis: xaxis, yaxis: yaxis}, **kwargs)\n",
    "    \n",
    "    if line_labels:\n",
    "        for c, label in enumerate(line_labels):\n",
    "            fig.data[c].name = label\n",
    "\n",
    "    fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generating lists of a fixed length parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data generation\n",
    "import random\n",
    "\n",
    "FIXED_LENGTH = 10\n",
    "\n",
    "def generateLists(n, training=True):\n",
    "    output = []\n",
    "    for _ in range(n):\n",
    "        curr = []\n",
    "        for _ in range(FIXED_LENGTH):\n",
    "            if training:\n",
    "                curr.append(random.randint(0, D_VOCAB - 1))\n",
    "            else:\n",
    "                curr.append(random.randint(D_VOCAB // 2, D_VOCAB - 1))\n",
    "            # curr.append(random.randint(0, 100))\n",
    "\n",
    "        # maximum = max(curr)\n",
    "        # output.append((curr, maximum))\n",
    "        output.append(curr)\n",
    "\n",
    "    output = torch.tensor(output)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generating data function with separate training and testing data so it doesn't see what\n",
    "it's being tested on. Data generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def separate_data(n, split=0.7):\n",
    "    output = []\n",
    "    for i in range(D_VOCAB):\n",
    "        for j in range(D_VOCAB):\n",
    "            curr = [i, j]\n",
    "            output.append(curr)\n",
    "\n",
    "    random.shuffle(output)\n",
    "\n",
    "    split_index = int(len(output) * split)\n",
    "    # return training, testing\n",
    "    return torch.tensor(output[:split_index]), torch.tensor(output[split_index:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def output_data(data, batch_size=128):\n",
    "    for i in range(0, len(data), batch_size):\n",
    "        yield data[i:i + batch_size]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Attempt at cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def raw_generate():\n",
    "    output = []\n",
    "    for i in range(D_VOCAB):\n",
    "        for j in range(D_VOCAB):\n",
    "            curr = [i, j]\n",
    "            output.append(curr)\n",
    "\n",
    "    random.shuffle(output)\n",
    "    return output\n",
    "\n",
    "def cross_val_generate(output, epoch, total_epoch):\n",
    "    testing_size = int(len(output) / total_epoch)\n",
    "    split_index = testing_size * epoch\n",
    "    # return training, testing\n",
    "    testing = torch.tensor(output[split_index:split_index + testing_size])\n",
    "    training = torch.tensor(output[:split_index] + output[split_index + testing_size:])\n",
    "    return training, testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generating variable length lists without separation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[48, 45, 61, 29,  0, 55, 61,  0,  0,  0],\n",
       "        [58, 24, 15, 27, 34, 32, 14, 24,  5,  0],\n",
       "        [62, 53, 37, 18, 53,  0,  0,  0,  0,  0],\n",
       "        [ 1, 27,  5, 57, 10, 47, 46,  6, 10,  0],\n",
       "        [38, 62, 28,  2, 16,  0,  0,  0,  0,  0],\n",
       "        [28, 23,  5, 32, 49, 35, 41,  0,  0,  0],\n",
       "        [16, 21, 16, 30, 61, 13, 47,  9, 32,  0],\n",
       "        [30, 17, 34, 16, 31, 21, 50,  0,  0,  0],\n",
       "        [28, 25, 26,  5,  0,  0,  0,  0,  0,  0],\n",
       "        [36,  0,  0,  0,  0,  0,  0,  0,  0,  0]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def generateVariableLists(n, seen, training=True):\n",
    "    output = []\n",
    "    index = 0\n",
    "\n",
    "    while index < n:\n",
    "        i = 0\n",
    "        j = random.randint(1, FIXED_LENGTH)\n",
    "        curr = []\n",
    "        while i < j:\n",
    "            curr.append(random.randint(0, D_VOCAB - 1))\n",
    "            i += 1\n",
    "        while i < FIXED_LENGTH:\n",
    "            curr.append(0)\n",
    "            i += 1\n",
    "\n",
    "        if training:\n",
    "            seen.add(tuple(curr))\n",
    "            output.append(curr)\n",
    "            index += 1\n",
    "        else:\n",
    "            if tuple(curr) not in seen:\n",
    "                output.append(curr)\n",
    "                index += 1\n",
    "    \n",
    "    output = torch.tensor(output)\n",
    "    return output\n",
    "\n",
    "generateVariableLists(10, set())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model paramters. We are using one layer, one attention head (which pays to the \n",
    "tokens in contex to another), the dimensions of the model, dimension of the head,\n",
    "vocab is the size of the logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model setup\n",
    "cfg = HookedTransformerConfig(\n",
    "    d_model=D_MODEL,\n",
    "    n_layers=N_LAYERS,\n",
    "    n_heads=N_HEADS,\n",
    "    d_head=D_HEAD,\n",
    "    n_ctx=FIXED_LENGTH,\n",
    "    d_vocab=D_VOCAB,\n",
    "    act_fn=\"relu\",\n",
    "    seed=SEED,\n",
    "    device=DEVICE,\n",
    "    attn_only=True\n",
    ")\n",
    "\n",
    "# hooked transformer used for interpretation later\n",
    "model = HookedTransformer(cfg, move_to_device=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_function(logits, tokens, return_per_token=True, print_tokens=False):\n",
    "    # we take the last element of the logits to make the next prediction\n",
    "    logits = logits[:, -1, :]\n",
    "    answer = torch.max(tokens, dim=1)[0]\n",
    "    log_prob = logits.log_softmax(-1)\n",
    "    if print_tokens:\n",
    "        print(\"tokens\", tokens)\n",
    "        print(\"predicted\", torch.argmax(logits, dim=-1))\n",
    "    # shape is (batch_size, 1) which represents probabilities \n",
    "    # of the correct answer\n",
    "    output_prob = log_prob.gather(-1, answer.unsqueeze(-1))\n",
    "    if return_per_token:\n",
    "        return -1 * output_prob.squeeze()\n",
    "    return -1 * output_prob.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(logits, tokens, return_per_token=False):\n",
    "    logits = logits[:, -1, :]\n",
    "    predicted = torch.argmax(logits, dim=1)\n",
    "    answer = torch.max(tokens, dim=1)[0]\n",
    "    if return_per_token:\n",
    "        return (predicted == answer).float()\n",
    "    return (predicted == answer).float().mean().item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.tensor([[1], [2], [3], [4]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, n_epochs, batch_size, batches_per, sequence_length=2):\n",
    "    lr = 1e-3\n",
    "    betas = (0.9, 0.999)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=lr, betas=betas)\n",
    "\n",
    "    seen = set()\n",
    "    train_losses = []\n",
    "    for epoch in range(n_epochs):\n",
    "        epoch_losses = []\n",
    "        for _ in range(batches_per):\n",
    "            tokens = generateVariableLists(batch_size, seen, training=True)\n",
    "            logits = model(tokens)\n",
    "            # print(tokens.shape)\n",
    "            # print(logits.shape)\n",
    "            losses = loss_function(logits, tokens, print_tokens=False)\n",
    "            losses.mean().backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "            epoch_losses.extend(losses.detach())\n",
    "\n",
    "        train_losses.append(np.mean(epoch_losses))\n",
    "        if epoch % 10 == 0:\n",
    "            print(f\"Epoch {epoch}, train loss: {train_losses[-1]}\")\n",
    "\n",
    "    model.eval()\n",
    "    # might want to create the training and testing set beforehand\n",
    "    test_data = generateVariableLists(1280, seen, training=False)\n",
    "    logits = model(test_data)\n",
    "    acc = accuracy(logits, test_data, return_per_token=False)\n",
    "\n",
    "    print(f\"Test accuracy: {acc}\")\n",
    "\n",
    "    return train_losses, seen\n",
    "                "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Separate training and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model2(model, n_epochs, batch_size, batches_per, sequence_length=2):\n",
    "    lr = 1e-3\n",
    "    betas = (0.9, 0.999)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=lr, betas=betas)\n",
    "\n",
    "    train_losses = []\n",
    "    training_data, testing_data = separate_data(batch_size)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        data_generator = output_data(training_data, batch_size)\n",
    "        epoch_losses = []\n",
    "        for _ in range(batches_per):\n",
    "            tokens = next(data_generator)\n",
    "            logits = model(tokens)\n",
    "            # print(tokens.shape)\n",
    "            # print(logits.shape)\n",
    "            losses = loss_function(logits, tokens, print_tokens=False)\n",
    "            losses.mean().backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "            epoch_losses.extend(losses.detach())\n",
    "\n",
    "        train_losses.append(np.mean(epoch_losses))\n",
    "        if epoch % 10 == 0:\n",
    "            print(f\"Epoch {epoch}, train loss: {train_losses[-1]}\")\n",
    "\n",
    "    model.eval()\n",
    "    # might want to create the training and testing set beforehand\n",
    "    logits = model(testing_data)\n",
    "    acc = accuracy(logits, testing_data, return_per_token=False)\n",
    "\n",
    "    print(f\"Test accuracy: {acc}\")\n",
    "\n",
    "    return train_losses\n",
    "                "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_val(model, n_epochs, batch_size, batches_per, sequence_length=2):\n",
    "    lr = 1e-3\n",
    "    betas = (0.9, 0.999)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=lr, betas=betas)\n",
    "\n",
    "    train_losses = []\n",
    "    all_data = raw_generate()\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        tokens, test = cross_val_generate(all_data, epoch, n_epochs - 1)\n",
    "        epoch_losses = []\n",
    "        for _ in range(batches_per):\n",
    "            logits = model(tokens)\n",
    "            # print(tokens.shape)\n",
    "            # print(logits.shape)\n",
    "            losses = loss_function(logits, tokens, print_tokens=False)\n",
    "            losses.mean().backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "            epoch_losses.extend(losses.detach())\n",
    "\n",
    "        train_losses.append(np.mean(epoch_losses))\n",
    "\n",
    "        model.eval()\n",
    "        logits = model(test)\n",
    "        acc = accuracy(logits, test, return_per_token=False)\n",
    "        if epoch % 10 == 0:\n",
    "            print(f\"Epoch {epoch}, train loss: {train_losses[-1]}\")\n",
    "            print(f\"Test accuracy: {acc}\")\n",
    "\n",
    "    return train_losses\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, train loss: 4.30193567276001\n",
      "Epoch 10, train loss: 1.6819137334823608\n",
      "Epoch 20, train loss: 0.4722824692726135\n",
      "Epoch 30, train loss: 0.2083248347043991\n",
      "Epoch 40, train loss: 0.13007839024066925\n",
      "Epoch 50, train loss: 0.08391229808330536\n",
      "Epoch 60, train loss: 0.07576825469732285\n",
      "Epoch 70, train loss: 0.043337367475032806\n",
      "Epoch 80, train loss: 0.031486447900533676\n",
      "Epoch 90, train loss: 0.051958613097667694\n",
      "Epoch 100, train loss: 0.020406896248459816\n",
      "Epoch 110, train loss: 0.02451227605342865\n",
      "Epoch 120, train loss: 0.02140043117105961\n",
      "Epoch 130, train loss: 0.022142618894577026\n",
      "Epoch 140, train loss: 0.019426560029387474\n",
      "Epoch 150, train loss: 0.01185769122093916\n",
      "Epoch 160, train loss: 0.008229217492043972\n",
      "Epoch 170, train loss: 0.024861464276909828\n",
      "Epoch 180, train loss: 0.012377142906188965\n",
      "Epoch 190, train loss: 0.009810050949454308\n",
      "Test accuracy: 0.9937499761581421\n"
     ]
    }
   ],
   "source": [
    "losses, seen = train_model(model, 200, 128, 10, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertemplate": "Epoch=%{x}<br>Loss=%{y}<extra></extra>",
         "legendgroup": "",
         "line": {
          "color": "#636efa",
          "dash": "solid"
         },
         "marker": {
          "symbol": "circle"
         },
         "mode": "lines",
         "name": "",
         "orientation": "v",
         "showlegend": false,
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119,
          120,
          121,
          122,
          123,
          124,
          125,
          126,
          127,
          128,
          129,
          130,
          131,
          132,
          133,
          134,
          135,
          136,
          137,
          138,
          139,
          140,
          141,
          142,
          143,
          144,
          145,
          146,
          147,
          148,
          149,
          150,
          151,
          152,
          153,
          154,
          155,
          156,
          157,
          158,
          159,
          160,
          161,
          162,
          163,
          164,
          165,
          166,
          167,
          168,
          169,
          170,
          171,
          172,
          173,
          174,
          175,
          176,
          177,
          178,
          179,
          180,
          181,
          182,
          183,
          184,
          185,
          186,
          187,
          188,
          189,
          190,
          191,
          192,
          193,
          194,
          195,
          196,
          197,
          198,
          199
         ],
         "xaxis": "x",
         "y": [
          4.30193567276001,
          3.8787522315979004,
          3.728400707244873,
          3.5652756690979004,
          3.4690747261047363,
          3.449145555496216,
          3.153468608856201,
          2.880980968475342,
          2.5383307933807373,
          2.1352832317352295,
          1.6819137334823608,
          1.4663712978363037,
          1.2879679203033447,
          1.0934646129608154,
          0.9483877420425415,
          0.8695688247680664,
          0.7718005180358887,
          0.6666849851608276,
          0.5544798374176025,
          0.5508043766021729,
          0.4722824692726135,
          0.3907124400138855,
          0.4150776267051697,
          0.35369205474853516,
          0.3388640582561493,
          0.31397175788879395,
          0.2599876821041107,
          0.2671695947647095,
          0.23265612125396729,
          0.18793508410453796,
          0.2083248347043991,
          0.1878570169210434,
          0.1977250725030899,
          0.1728140413761139,
          0.16819624602794647,
          0.169254332780838,
          0.1301461160182953,
          0.1473417580127716,
          0.13829544186592102,
          0.13897296786308289,
          0.13007839024066925,
          0.10563375055789948,
          0.09224239736795425,
          0.11079444736242294,
          0.10068388283252716,
          0.10384192317724228,
          0.1245357021689415,
          0.10394029319286346,
          0.0947609394788742,
          0.1058366522192955,
          0.08391229808330536,
          0.08998323231935501,
          0.0853278860449791,
          0.08256147056818008,
          0.07847091555595398,
          0.0657789334654808,
          0.05692298337817192,
          0.06038955599069595,
          0.07179594784975052,
          0.0695221796631813,
          0.07576825469732285,
          0.060957156121730804,
          0.06494773179292679,
          0.07184450328350067,
          0.07037614285945892,
          0.06791291385889053,
          0.0646207183599472,
          0.07006390392780304,
          0.07791433483362198,
          0.05080381780862808,
          0.043337367475032806,
          0.05483246594667435,
          0.045702047646045685,
          0.045005735009908676,
          0.03502943366765976,
          0.041631560772657394,
          0.03427579998970032,
          0.0409325435757637,
          0.039008062332868576,
          0.03530091792345047,
          0.031486447900533676,
          0.04631071537733078,
          0.040702879428863525,
          0.04585547000169754,
          0.04379379004240036,
          0.03742997720837593,
          0.02462633140385151,
          0.03276144340634346,
          0.052198637276887894,
          0.04495230317115784,
          0.051958613097667694,
          0.03521483764052391,
          0.02775125578045845,
          0.03717530891299248,
          0.025045419111847878,
          0.03067828342318535,
          0.0284056905657053,
          0.020040538161993027,
          0.029957126826047897,
          0.03017370030283928,
          0.020406896248459816,
          0.015755191445350647,
          0.029715541750192642,
          0.02012038417160511,
          0.022821184247732162,
          0.043951503932476044,
          0.024606427177786827,
          0.03559959679841995,
          0.02700411155819893,
          0.0215332992374897,
          0.02451227605342865,
          0.03353553265333176,
          0.02511315979063511,
          0.03736823797225952,
          0.03367609903216362,
          0.024983752518892288,
          0.02631094679236412,
          0.020614493638277054,
          0.019327476620674133,
          0.02286546118557453,
          0.02140043117105961,
          0.025250231847167015,
          0.031015027314424515,
          0.029609162360429764,
          0.02341802977025509,
          0.013225702568888664,
          0.016072701662778854,
          0.018255790695548058,
          0.0156357791274786,
          0.019704289734363556,
          0.022142618894577026,
          0.020822346210479736,
          0.016655076295137405,
          0.015709687024354935,
          0.019137347117066383,
          0.018347419798374176,
          0.02661975286900997,
          0.02603534795343876,
          0.031181324273347855,
          0.025434643030166626,
          0.019426560029387474,
          0.01698937639594078,
          0.02641211822628975,
          0.017895495519042015,
          0.014307038858532906,
          0.021836355328559875,
          0.020286468788981438,
          0.023491378873586655,
          0.018524710088968277,
          0.021852392703294754,
          0.01185769122093916,
          0.02032320946455002,
          0.029694801196455956,
          0.013715964742004871,
          0.017901364713907242,
          0.015727784484624863,
          0.016439590603113174,
          0.015651559457182884,
          0.03271859139204025,
          0.01912093721330166,
          0.008229217492043972,
          0.026004869490861893,
          0.009500542655587196,
          0.010591565631330013,
          0.009760777465999126,
          0.010578676126897335,
          0.01598069630563259,
          0.02032662369310856,
          0.01952623389661312,
          0.011463822796940804,
          0.024861464276909828,
          0.01616603508591652,
          0.011398708447813988,
          0.009322268888354301,
          0.018132522702217102,
          0.018701547756791115,
          0.017083778977394104,
          0.01607859507203102,
          0.02227359637618065,
          0.017456134781241417,
          0.012377142906188965,
          0.015275207348167896,
          0.014235991053283215,
          0.01079520396888256,
          0.009032627567648888,
          0.010237986221909523,
          0.017013927921652794,
          0.00724058598279953,
          0.010712264105677605,
          0.00770718464627862,
          0.009810050949454308,
          0.014076976105570793,
          0.010738514363765717,
          0.018088890239596367,
          0.009233750402927399,
          0.020540164783596992,
          0.015830112621188164,
          0.01417542714625597,
          0.014362843707203865,
          0.009957184083759785
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "legend": {
         "tracegroupgap": 0
        },
        "margin": {
         "t": 60
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "Epoch"
         }
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "title": {
          "text": "Loss"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "line(losses, xaxis=\"Epoch\", yaxis=\"Loss\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualizing Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training_data, testing_data = separate_data(128)\n",
    "# train_data_gen = output_data(training_data)\n",
    "# tokens = next(train_data_gen)\n",
    "# logits, cache = model.run_with_cache(tokens)\n",
    "\n",
    "tokens = generateVariableLists(128, seen, training=False)\n",
    "special_token = [1, 3, 11, 7, 5, 0, 0, 35, 0, 0]\n",
    "tokens = torch.cat([torch.tensor([special_token]), tokens], dim=0)\n",
    "\n",
    "logits, cache = model.run_with_cache(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([35, 49, 56, 45, 42, 62, 62, 52, 53, 61, 60, 63, 59, 61, 56, 51, 47, 63,\n",
      "        59, 41, 52, 56, 57, 59, 60, 48, 63, 60, 56, 54, 57, 63, 29, 46, 55, 56,\n",
      "        61, 59, 51, 56, 62, 53, 45, 59, 57, 59, 54, 58, 36, 59, 53, 57, 62, 58,\n",
      "        60, 63, 60, 60, 17, 58, 46, 47, 62, 60, 60, 59, 41, 61, 49, 55, 61, 51,\n",
      "        52, 58, 41, 52, 58, 58, 54, 62, 61, 60, 62, 61, 62, 57, 62, 61, 60, 63,\n",
      "        52, 55, 58, 50, 47, 60, 60, 61, 52, 51, 54, 48, 63, 33, 41, 56, 59, 49,\n",
      "        52, 42, 61, 52, 62, 55, 42, 46, 62, 49, 50, 62, 60, 59, 61, 57, 54, 47,\n",
      "        60, 55, 48])\n"
     ]
    }
   ],
   "source": [
    "attention_pattern = cache[\"pattern\", 0, \"attn\"]\n",
    "\n",
    "# prints the first answer predicted by logits\n",
    "print(torch.argmax(logits[:, -1, :], dim=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div id=\"circuits-vis-a9fb8256-870e\" style=\"margin: 15px 0;\"/>\n",
       "    <script crossorigin type=\"module\">\n",
       "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
       "    render(\n",
       "      \"circuits-vis-a9fb8256-870e\",\n",
       "      AttentionHeads,\n",
       "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31769225001335144, 0.6823077201843262, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0876765176653862, 0.16763296723365784, 0.7446905374526978, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02225734107196331, 0.08940236270427704, 0.5908180475234985, 0.29752230644226074, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1367548257112503, 0.19029246270656586, 0.4441041648387909, 0.15135324001312256, 0.07749536633491516, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12075667828321457, 0.19952644407749176, 0.35802215337753296, 0.21123796701431274, 0.0814567357301712, 0.029000021517276764, 0.0, 0.0, 0.0, 0.0], [0.07834180444478989, 0.20303833484649658, 0.3210757374763489, 0.18597999215126038, 0.11709367483854294, 0.04730715602636337, 0.04716334491968155, 0.0, 0.0, 0.0], [0.004737577401101589, 0.008753517642617226, 0.02410980872809887, 0.014947587624192238, 0.0028466740623116493, 0.000940344063565135, 0.001177302561700344, 0.9424871802330017, 0.0, 0.0], [0.16957055032253265, 0.1489390879869461, 0.1613147258758545, 0.07836516946554184, 0.08022820204496384, 0.056730225682258606, 0.07797747850418091, 0.15127940475940704, 0.07559515535831451, 0.0], [7.877918619669799e-07, 3.373214894963894e-06, 4.1086408600676805e-05, 8.278398127004039e-06, 4.4950596134185616e-07, 4.9957509240528e-09, 1.4584048635413183e-08, 0.9999459981918335, 3.961783345829417e-09, 1.2967364071414522e-08]]], \"tokens\": [\"1\", \"3\", \"11\", \"7\", \"5\", \"0\", \"0\", \"35\", \"0\", \"0\"]}\n",
       "    )\n",
       "    </script>"
      ],
      "text/plain": [
       "<circuitsvis.utils.render.RenderedHTML at 0x30b5f3250>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.attention.attention_heads(tokens=list(map(lambda t: str(t.item()), tokens[0])), attention=attention_pattern[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div id=\"circuits-vis-5a3672e3-2fe9\" style=\"margin: 15px 0;\"/>\n",
       "    <script crossorigin type=\"module\">\n",
       "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
       "    render(\n",
       "      \"circuits-vis-5a3672e3-2fe9\",\n",
       "      AttentionPatterns,\n",
       "      {\"tokens\": [\"1\", \"3\", \"11\", \"7\", \"5\", \"0\", \"0\", \"35\", \"0\", \"0\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31769225001335144, 0.6823077201843262, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0876765176653862, 0.16763296723365784, 0.7446905374526978, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02225734107196331, 0.08940236270427704, 0.5908180475234985, 0.29752230644226074, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1367548257112503, 0.19029246270656586, 0.4441041648387909, 0.15135324001312256, 0.07749536633491516, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12075667828321457, 0.19952644407749176, 0.35802215337753296, 0.21123796701431274, 0.0814567357301712, 0.029000021517276764, 0.0, 0.0, 0.0, 0.0], [0.07834180444478989, 0.20303833484649658, 0.3210757374763489, 0.18597999215126038, 0.11709367483854294, 0.04730715602636337, 0.04716334491968155, 0.0, 0.0, 0.0], [0.004737577401101589, 0.008753517642617226, 0.02410980872809887, 0.014947587624192238, 0.0028466740623116493, 0.000940344063565135, 0.001177302561700344, 0.9424871802330017, 0.0, 0.0], [0.16957055032253265, 0.1489390879869461, 0.1613147258758545, 0.07836516946554184, 0.08022820204496384, 0.056730225682258606, 0.07797747850418091, 0.15127940475940704, 0.07559515535831451, 0.0], [7.877918619669799e-07, 3.373214894963894e-06, 4.1086408600676805e-05, 8.278398127004039e-06, 4.4950596134185616e-07, 4.9957509240528e-09, 1.4584048635413183e-08, 0.9999459981918335, 3.961783345829417e-09, 1.2967364071414522e-08]]]}\n",
       "    )\n",
       "    </script>"
      ],
      "text/plain": [
       "<circuitsvis.utils.render.RenderedHTML at 0x30b8ed710>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.attention.attention_patterns(tokens=list(map(lambda t: str(t.item()), tokens[0])), attention=attention_pattern[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div id=\"circuits-vis-c193aa55-12ad\" style=\"margin: 15px 0;\"/>\n",
       "    <script crossorigin type=\"module\">\n",
       "    import { render, AttentionHeads } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
       "    render(\n",
       "      \"circuits-vis-c193aa55-12ad\",\n",
       "      AttentionHeads,\n",
       "      {\"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9146789908409119, 0.08532103896141052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9634140729904175, 0.00489401537925005, 0.03169197961688042, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0032953552436083555, 1.1968140825047158e-05, 6.095872595324181e-05, 0.9966316819190979, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08568061143159866, 0.29422932863235474, 0.26776638627052307, 0.03239605948328972, 0.3199276626110077, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0857556015253067, 0.004259856417775154, 0.013169920071959496, 0.38617613911628723, 0.008980854414403439, 0.5016576647758484, 0.0, 0.0, 0.0, 0.0], [0.033156633377075195, 0.003177707316353917, 0.003816588781774044, 0.37403586506843567, 0.009246725589036942, 0.5579918026924133, 0.01857469603419304, 0.0, 0.0, 0.0], [0.05968758836388588, 0.007923935540020466, 0.029319291934370995, 0.3152613341808319, 0.016939383000135422, 0.5134825706481934, 0.02630983106791973, 0.031076082959771156, 0.0, 0.0], [0.20260140299797058, 0.09425865113735199, 0.1581762284040451, 0.11995865404605865, 0.1193455159664154, 0.07318205386400223, 0.06833017617464066, 0.09738992154598236, 0.06675748527050018, 0.0], [1.8172067939303815e-06, 1.1731374552348406e-11, 7.38251237919485e-10, 0.12981970608234406, 5.999730712247242e-10, 0.8701785206794739, 3.545287796669072e-08, 1.0429539898382245e-08, 3.079805291003601e-12, 1.0080550109925479e-11]]], \"tokens\": [\"23\", \"0\", \"4\", \"46\", \"2\", \"49\", \"8\", \"12\", \"0\", \"0\"]}\n",
       "    )\n",
       "    </script>"
      ],
      "text/plain": [
       "<circuitsvis.utils.render.RenderedHTML at 0x30b61f1d0>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.attention.attention_heads(tokens=list(map(lambda t: str(t.item()), tokens[1])), attention=attention_pattern[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div id=\"circuits-vis-4addb2a2-84a0\" style=\"margin: 15px 0;\"/>\n",
       "    <script crossorigin type=\"module\">\n",
       "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
       "    render(\n",
       "      \"circuits-vis-4addb2a2-84a0\",\n",
       "      AttentionPatterns,\n",
       "      {\"tokens\": [\"23\", \"0\", \"4\", \"46\", \"2\", \"49\", \"8\", \"12\", \"0\", \"0\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9146789908409119, 0.08532103896141052, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9634140729904175, 0.00489401537925005, 0.03169197961688042, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0032953552436083555, 1.1968140825047158e-05, 6.095872595324181e-05, 0.9966316819190979, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08568061143159866, 0.29422932863235474, 0.26776638627052307, 0.03239605948328972, 0.3199276626110077, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0857556015253067, 0.004259856417775154, 0.013169920071959496, 0.38617613911628723, 0.008980854414403439, 0.5016576647758484, 0.0, 0.0, 0.0, 0.0], [0.033156633377075195, 0.003177707316353917, 0.003816588781774044, 0.37403586506843567, 0.009246725589036942, 0.5579918026924133, 0.01857469603419304, 0.0, 0.0, 0.0], [0.05968758836388588, 0.007923935540020466, 0.029319291934370995, 0.3152613341808319, 0.016939383000135422, 0.5134825706481934, 0.02630983106791973, 0.031076082959771156, 0.0, 0.0], [0.20260140299797058, 0.09425865113735199, 0.1581762284040451, 0.11995865404605865, 0.1193455159664154, 0.07318205386400223, 0.06833017617464066, 0.09738992154598236, 0.06675748527050018, 0.0], [1.8172067939303815e-06, 1.1731374552348406e-11, 7.38251237919485e-10, 0.12981970608234406, 5.999730712247242e-10, 0.8701785206794739, 3.545287796669072e-08, 1.0429539898382245e-08, 3.079805291003601e-12, 1.0080550109925479e-11]]]}\n",
       "    )\n",
       "    </script>"
      ],
      "text/plain": [
       "<circuitsvis.utils.render.RenderedHTML at 0x30b8ed4d0>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.attention.attention_patterns(tokens=list(map(lambda t: str(t.item()), tokens[1])), attention=attention_pattern[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div id=\"circuits-vis-cc5d8c90-235d\" style=\"margin: 15px 0;\"/>\n",
       "    <script crossorigin type=\"module\">\n",
       "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
       "    render(\n",
       "      \"circuits-vis-cc5d8c90-235d\",\n",
       "      AttentionPatterns,\n",
       "      {\"tokens\": [\"1\", \"40\", \"18\", \"16\", \"24\", \"56\", \"9\", \"14\", \"28\", \"0\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9860144853591919, 0.013985546305775642, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [4.616397200152278e-05, 0.9981456995010376, 0.0018081970047205687, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006991684436798096, 0.9119319915771484, 0.04961169511079788, 0.031464576721191406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.030866330489516258, 0.739623486995697, 0.08447183668613434, 0.05951305106282234, 0.08552513271570206, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04369174316525459, 0.2201368659734726, 0.0864860862493515, 0.07085125148296356, 0.05350326746702194, 0.5253307223320007, 0.0, 0.0, 0.0, 0.0], [0.006695693824440241, 0.11470203846693039, 0.039848897606134415, 0.04747334495186806, 0.06255083531141281, 0.7071337699890137, 0.021595370024442673, 0.0, 0.0, 0.0], [1.6454268916277215e-05, 0.015542064793407917, 0.00019981390505563468, 0.00016216584481298923, 0.0006317686638794839, 0.9831861257553101, 0.00019934245210606605, 6.227337144082412e-05, 0.0, 0.0], [0.00031585581018589437, 0.04275156930088997, 0.0015905555337667465, 0.0005207398207858205, 0.007276721298694611, 0.9404432773590088, 0.0006601776694878936, 0.0012817103415727615, 0.00515931099653244, 0.0], [3.4827509799717493e-12, 2.8931337510584854e-05, 1.1976931668655766e-09, 1.6595362584137519e-10, 2.8193477774607345e-08, 0.999970555305481, 6.972490917966923e-10, 5.57889179297888e-10, 4.736851053621649e-07, 5.732745414445059e-14]]]}\n",
       "    )\n",
       "    </script>"
      ],
      "text/plain": [
       "<circuitsvis.utils.render.RenderedHTML at 0x30b7b8b50>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.attention.attention_patterns(tokens=list(map(lambda t: str(t.item()), tokens[2])), attention=attention_pattern[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvgAAAMWCAYAAACEG/KlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABvXUlEQVR4nO3dfXRV1b3v/09AEh5kZxuEhEh4EgpYBRUUg7SlmorU45WfDKte7xAdHv3VAVbEHlvOsOBDK304Fuq5KLW1Uu8ph5bzu9DaVrwUC94eAQXLqR4rFUGJYoKiSSCWBJP9+8Pj1hTY+xNYWVlM3q8xMoZmf1lz7rke9szKzPoUZDKZjAAAAAAEoUtndwAAAABAdJjgAwAAAAFhgg8AAAAEhAk+AAAAEBAm+AAAAEBAmOADAAAAAWGCDwAAAASECT4AAAAQkBM6uwMAAAAI3/79+9Xc3NzZ3ThIYWGhunfv3tndiBQTfAAAAHSo/fv3a8iQIaqpqensrhykrKxMO3bsCGqSzwQfAAAAHaq5uVk1NTWqrq5WKpXq7O5kNTQ0qKKiQs3NzUzwAQAAgPZKpVKJmuCHigk+AAAAYvLBf30lRZL6Eh2eogMAAAAEhAk+AAAAEBCW6AAAACAmLNGJA3fwAQAAgIAwwQcAAAACwhIdAAAAxIQlOnHgDj4AAAAQECb4AAAAQEBYogMAAICYsEQnDtzBBwAAAALCBB8AAAAICBN8AAAAxKRFHy/TScJXS7t6P3/+fJ1zzjnq3bu3+vXrp6lTp2rr1q15/93y5cs1cuRIde/eXWeccYZ++9vftqvd9mKCDwAAABjWrVunGTNmaMOGDVq9erUOHDigiy66SI2NjYf9N88884yuvvpq3XDDDfrjH/+oqVOnaurUqXrxxRc7rJ8FmUwm02FbBwAAwHGvoaFBxcXFqq/frlSqd2d3J6uhYa+Ki4eqvr5eqVSq3f/+7bffVr9+/bRu3Tp99rOfPWTNlVdeqcbGRv3617/Ofu+8887TmWeeqcWLFx9x33PhDj4AAABi0tlLcg71deTq6+slSSUlJYetWb9+vaqqqtp8b/LkyVq/fv1RtZ0Lj8kEAADAca2hoaHN/xcVFamoqCjnv2ltbdWsWbN0/vnn6/TTTz9sXU1NjUpLS9t8r7S0VDU1NUfe4Ty4gw8AAIDjWkVFhYqLi7Nf8+fPz/tvZsyYoRdffFHLli2LoYftwx18AAAAxCSZQVfV1dVt1uDnu3s/c+ZM/frXv9bTTz+tAQMG5KwtKytTbW1tm+/V1taqrKzsCPucH3fwAQAAcFxLpVJtvg43wc9kMpo5c6ZWrFihp556SkOGDMm77crKSq1Zs6bN91avXq3KyspI+n4o3MEHAAAADDNmzNDSpUv1y1/+Ur17986uoy8uLlaPHj0kSddee61OOeWU7DKfW2+9VZ/73Od0//3365JLLtGyZcu0adMmPfzwwx3WTyb4AAAAiEkyl+i4HnroIUnSpEmT2nz/0Ucf1XXXXSdJ2rlzp7p0+XiRzIQJE7R06VLdeeed+sd//EcNHz5cK1euzPmHuUeL5+ADAACgQ338HPw/JfA5+KOP+Dn4ScUafAAAACAgLNEBAABATFr+6yspktSX6HAHHwAAAAgIE3wAAAAgICzRAQAAQExalKyn6LBEBwAAAEDCMcEHAAAAAsISHQAAAMTk2A66OlZwBx8AAAAICBN8AAAAICAs0QEAAEBMWKITB+7gAwAAAAFhgg8AAAAEhCU6AAAAiAlLdOLAHXwAAAAgIEzwAQAAgICwRAcAAAAxaVGylsW0dHYHOgR38AEAAICAMMEHAAAAAsISHQAAAMSEp+jEgTv4AAAAQECY4AMAAAABYYkOAAAAYsISnThwBx8AAAAICBN8AAAAICAs0QEAAEBMWKITB+7gAwAAAAFhgg8AAAAEhCU6AAAAiAlLdOLAHXwAAAAgIEzwAQAAgICwRAcAAAAxaVGylsW0dHYHOgR38AEAAICAMMEHAAAAAsISHQAAAMSEp+jEgTv4AAAAQECY4AMAAAABYYkOAAAAYsISnThwBx8AAAAICBN8AAAAICAs0QEAAEBMWKITB+7gAwAAAAFhgg8AAAAEhCU6AAAAiAlLdOLAHXwAAAAgIEzwAQAAgICwRAcAAAAxaVGylsW0dHYHOgR38AEAAICAMMEHAAAAAsISHQAAAMSkRclaFpOkvkSHO/gAAABAQJjgAwAAAAFhiQ4AAABiQtBVHLiDDwAAAASECT4AAAAQEJboAAAAICYs0YkDd/ABAACAgDDBBwAAAALCEh0AAADEpEXJWhZD0BUAAACAhGOCDwAAAAQkcUt0WltbtWvXLvXu3VsFBQWd3R0AAIBjRiaT0d69e1VeXq4uXZJ4H5en6MShwyb4ixYt0ve+9z3V1NRozJgx+ud//mede+65ef/drl27VFFR0VHdAgAACF51dbUGDBjQ2d1AJ+mQCf7Pf/5zzZ49W4sXL9b48eO1cOFCTZ48WVu3blW/fv1y/tvevXtLkqq7SalcN/BLzM44fzvh1KSNmjqjRpK6GzXdjJomsz1nWz2Mmr+a7Tk3DHoaNU6/9xs1kjdWRUaNOwYnmnX5FJt19UaNc9w5VwRnv0jemDvbSuLfP7Wadc77c44Vt733jZqUUdNgtue8P6fGOe7eM2okqZdRU2jUuDf14v49uHMsONdXZzvuue5cO5323OvmPqPGGQOn31F+zjrHSm+zvRyfDQ37pYp7P55P4fjUIZem73//+7rxxht1/fXXS5IWL16s3/zmN/rJT36ir3/96zn/7UfLclIFeSb47m+dMhHVOO25fYpqW1G21zWB7UVVI8U/Bm6/8nHP0KjGymnP7ZMzQXK2lcSVeu4PHVH9wBR3e+4+jup4ibtP7sTVEeW2HFFNzKOc4Ds/MDnHsLMdyeuXsy1ngu/MD6Tofth1+iRZN2ySu8yZJTpxiHxxVnNzszZv3qyqqqqPG+nSRVVVVVq/fn3UzQEAAAD4hMjv4L/zzjtqaWlRaWlpm++Xlpbq5ZdfPqi+qalJTU0f/w6socH93TAAAACAv9Xpf149f/58FRcXZ7/4A1sAAIBQfZDAr/BEPsE/+eST1bVrV9XW1rb5fm1trcrKyg6qnzNnjurr67Nf1dXVUXcJAAAAOG5EPsEvLCzU2LFjtWbNmuz3WltbtWbNGlVWVh5UX1RUpFQq1eYLAAAAwJHpkKfozJ49W9OnT9e4ceN07rnnauHChWpsbMw+VcfyKeV+6of7l+YHjBrn6SLOIwYP/gXFoTk/VjlPOGg223OeJuD8dX+j2Z4zVs7jLZ394j4FzNmW8/5OMdtztuU8zs/dx86x5/TJGc+9Ro0U3ZM1onxSkrP/nHF626iRvPfnXKPcp+g4++8No8Z9dLbzWE7nWuaMgXNdkbxHy75r1LjXFuda5vTdGQPJu1Y723L2nXv9SRs1zjHsXlucx2Q714PXzPai4lzz3WtLrm25+63TJG1ZTJL6Ep0OmeBfeeWVevvttzV37lzV1NTozDPP1KpVqw76w1sAAAAA0eqwiI6ZM2dq5syZHbV5AAAAAIcQdwYfAAAAjlstStaymCTGpR+9Tn9MJgAAAIDoMMEHAAAAAsISHQAAAMTkA/mPR4tDkpYLRYc7+AAAAEBAmOADAAAAAWGJDgAAAGLCEp04JHeC//8od/qfk8InRZcy+KZRkzZqJK/vPSPajhRdWqiToBglp706c1tOqqrz+yzneJK88XTSJp3jQJLqjZpyo8bpk5vYHFVCqzvmznnstOccdxVGjeSl4qaNmjqzvSajZphR46ZpOse5mzqeT5TpwVG2F9W57ibZOpwxcNJ83euPc446icbumoLtRo1zPXD2sZvg7rw/Q8bZL3maC/Ohj2gvlugAAAAAAUnuHXwAAAAEhiU6ceAOPgAAABAQJvgAAABAQFiiAwAAgJi0KFnLYsL8s2Tu4AMAAACGp59+WpdeeqnKy8tVUFCglStX5qxfu3atCgoKDvqqqanp0H4ywQcAAAAMjY2NGjNmjBYtWtSuf7d161a99dZb2a9+/fp1UA8/xBIdAAAAxOQDJev+cvuWC02ZMkVTpkxpdyv9+vVTOp1u9787Uomd4A+7N/fu/5y5nbRRU23UXGnU/F+jRpKGGjW9jBp31ZiT9+FkF71gttc3ovZ6GzVuDstLRo3Tb/fBXs77c/afu49PMWqcy6kz5m4ej7MtJ6vN2Y4kFRYbRc7JYNhT69X1cUKzBhk17qA77+81o8YNM3O8btSUGDVu0F46ovaccDxJajZqnAu6cwGSvHFIGzXOBcG94DknqXNsjjDbc8KgzjZqeqTMBh0jjZrz81YU6BKrta668PCvNTRIxc4FEXE688wz1dTUpNNPP1133XWXzj8///FwNBI7wQcAAADi0NDQ0Ob/i4qKVFR09FHc/fv31+LFizVu3Dg1NTXpxz/+sSZNmqSNGzfq7LOdn0SPDBN8AAAAxCSZS3QqKirafHfevHm66667jnrrI0aM0IgRH/96asKECXr11Ve1YMEC/a//9b+OevuHwwQfAAAAx7Xq6mqlUh8v24ri7v3hnHvuufrDH/7QYduXmOADAADgOJdKpdpM8DvSli1b1L9//w5tgwk+AAAAYpLMJTquffv2adu2bdn/37Fjh7Zs2aKSkhINHDhQc+bM0ZtvvqnHHntMkrRw4UINGTJEn/70p7V//379+Mc/1lNPPaX/83/+T6Tv4m8xwQcAAAAMmzZt0uc///ns/8+ePVuSNH36dC1ZskRvvfWWdu7cmX29ublZt99+u95880317NlTo0eP1u9+97s22+gITPABAAAAw6RJk5TJZA77+pIlS9r8/x133KE77rijg3t1MCb4AAAAiEmL/JSXOCSpL9FJ7AR/2zAplStkww0F2WvU7DdqjJCO6Q35a9xtWcEobsqTMwZGoMnlUZ4DaaPG2S9mAND5zhg4YS3uGDhhNE6IjBs04xxTTt+d8XSCfSTrGO7htNcYXXtR6VNuFjrHgXMt22W25xzDfYwa59yTvPfn9Mk5zt0xeNuocYKS3GAtp84ZT/f4da779UaNswTa7ZMzBs412D2v6owa59rS1fjQdpeK93o2f02JUbN3gdderrC2Jm8TCFuS/soBAAAAwFFK7B18AAAAhKZF7X1yTccKc4kOd/ABAACAgDDBBwAAAALCEh0AAADE5ANJBZ3diU9I0nKh6HAHHwAAAAgIE3wAAAAgICzRAQAAQExYohOHxE7wv7pNKszx+me2ettx8kWcjI5xRs1/GDWSNNKocbKE3NyXoUaNk2fyitlehVmXT6lR02pu6zWjxum3k9UiedlFznHXwwzbyRihLgVFxoacwCwnuEiSao2aXGEtH3EDh6IKM3MCcmqMGskLlXrfqBlktve6URNl6JIzDlEF+7nHweCItuU+Oc857pyLi9ueE2J0ulHjjqfDeX9OmJkb3uhc+J0PvrRRs8eokbwxcM6Fgf3MBj93+JcaDkgPrDS3g1CxRAcAAAAISGLv4AMAACA0LNGJA3fwAQAAgIAwwQcAAAACwhIdAAAAxIQlOnHgDj4AAAAQECb4AAAAQEBYogMAAICYtChZS3TcEIpjC3fwAQAAgIAk9g7+Pw2VUrl+/HB/4HISEp1kRyMB81POdlxG4t1Yd1tOv4zEzcHu+3OSQN3Ewijakhco6iQ79otyHztJkmaiqHUvJMq+O4YbNc557CS9St4x5WwrbdQ4UcWSd3w66bpOeqnk9avcqHnXbM85sZwE5Vyx5R9x4r0lqdqoKTZq3DFw0phfMGpOMdtzuLHj+VgXTknbjBrndqKboPy2UePsFycRN8rEX0fP3V5d4fLDv+ZeLxC0xE7wAQAAEJqkPbUmaf2JBkt0AAAAgIAwwQcAAAACwhIdAAAAxCRpS2KS1p9ocAcfAAAACAgTfAAAACAgLNEBAABATJK2JCZp/YkGd/ABAACAgCT2Dv6C7bmznoaa23FyOhqMmrONGifrQ/JyZpyfvNwsCydHx8l92Wq25+SLpIyaqPJxJOlNo8bNdHE4Y+BkIHU12+vqhLFEFdjibsd5g8620mZ7RjicFYa1y6hpNGokbwc6J2jabG+PUeP03W3PCU9z+hTlyRfVMeWefM5x5wSQuSFPTjicc0y57Tmiuv64x8Feo+Z0o8bZx26opvPB7ox5hdlewcDDv9bQKv3wDXNDCFViJ/gAAAAIjftTU1yS1p9osEQHAAAACAgTfAAAACAgLNEBAABATD6QlOnsTnwCS3QAAAAAJBwTfAAAACAgLNEBAABATFiiEwfu4AMAAAABSewd/Nv6SqlcP364YRBOEIsbWpPHQDuVyKhxAnncwKFWsy6P8+rMQifMwwlieduoccZJ0qecYJS0UeMcT5IVelJYZ2zH3XfOseDsFyeUyOUEXUW5HSdoxjnu3o1oO5J3bXGuZXVme05QkHMuuJyEPOc2knMeNxs1kjdWzjHl7mOHc4PwfXNbzrlea9Q41zLnnHK35bw/9zPN2TfbjZo6oybK648Tiubav/Pwr7kpmAhaYif4AAAACA1LdOLAEh0AAAAgIEzwAQAAgICwRAcAAAAxYYlOHLiDDwAAAASECT4AAAAQEJboAAAAICYtStYSnYieJZ4w3MEHAAAAAsIEHwAAAAhIcpfoDFfu3rmpf07inZMuF1VSqOSlWzrpeW5yrvPbpyhT+JwfG50+lRg1bvKhk/Dp7Bd3DJxkR+d4cZNz00aN8/6cPrm3BZxtOQ8vcMfAOV6cbTnprG76tXP9ccbAPe6iut65v7F2EjOdMY8yXdc5151xqjvKfnyS8/7MVG7ruu98fjjtOWnibnsO99ryilFzilHjXlscTp/6GjVRnAtR7Y8OwxKdOHAHHwAAAAgIE3wAAAAgIMldogMAAIDAfKBk3V9miQ4AAACAhGOCDwAAAASEJToAAACICUt04pCkEQYAAABwlJjgAwAAAAFJ7hKdryh3EMcecztOaIQTerLdqHECViQvXMMJ0nGDrpzQCyf8pijC9qIK8nKPg3RE7dWZ7TnhRQ434MhRZ9REGbDmBOk452eN2V6ZUeME9zhjnjZqJG/MnW254TdOuJgTDucGADmBg85+cewy65zj802jxv2tvXNMOWPu7mPn2uKcx9VGjRMeJ3lj7hxTboClMwYbjc04QW0m56PIOQxqzfbqcrzm5vB1HpboxKHdI/z000/r0ksvVXl5uQoKCrRy5co2r2cyGc2dO1f9+/dXjx49VFVVpVdecSLeAAAAABytdk/wGxsbNWbMGC1atOiQr3/3u9/VAw88oMWLF2vjxo3q1auXJk+erP37o8yEBgAAAHAo7V6iM2XKFE2ZMuWQr2UyGS1cuFB33nmnLrvsMknSY489ptLSUq1cuVJXXXXV0fUWAAAAx7AWJWtZTKazO9AhIl0EtWPHDtXU1Kiqqir7veLiYo0fP17r168/5L9pampSQ0NDmy8AAAAARybSCX5NzYd/DVdaWtrm+6WlpdnX/tb8+fNVXFyc/aqoqIiySwAAAMBxpdP/jHnOnDmqr6/PflVXO3/aDwAAgGPPBwn8Ck+kE/yysg+fh1Zb2/ZBT7W1tdnX/lZRUZFSqVSbLwAAAABHJtIJ/pAhQ1RWVqY1a9Zkv9fQ0KCNGzeqsrIyyqYAAAAAHEK7n6Kzb98+bdu2Lfv/O3bs0JYtW1RSUqKBAwdq1qxZ+uY3v6nhw4dryJAh+sY3vqHy8nJNnTq1Xe0suCp33sxEcztOZs3rRs3ZRo37tP+0UVNu1Lh5Q81GjZOP4+bMOHlYzl9aRPk39k5O2TCjxg0hGewMgvMGzXCxln35a7qebGwoyqArJ7jHCcNyw76MYBvrQHdCdNw+OcE9zoHnBgA5oUul+Uvs9+ccC066T6FR4xwrkheMNtKocS6ckndBd871YrM9Rzqi7TghkJJ3XjnXAzdYyzEof0lX5/h1romS+kV0Hp801r3vmj7sKw0NGan4PXM7neEDSQWd3YlPCPMpOu2e4G/atEmf//zns/8/e/ZsSdL06dO1ZMkS3XHHHWpsbNRNN92kuro6TZw4UatWrVL37lFGcgIAAAA4lHZP8CdNmqRM5vA/7RQUFOiee+7RPffcc1QdAwAAANB+nf4UHQAAAADRafcdfAAAAODIsAY/DtzBBwAAAALCBB8AAAAICEt0AAAAEI9Ma7JWxSSpLxHiDj4AAAAQECb4AAAAQEASu0Tntr5SKtePH046omQl7I3db2ynb/6SwU4Cpsvpk5mwZ6VNGu+vT53ZnrNvonp/ZUaNpDOc9ox+D3YTRWP+0blrVAmtzrHivjdnrJykTPe8iipLz9mOkxjrclJcTzG35SSBOvvF6ZMkNRo1zlg557qbctrHrMvHjdJ+16hxjmFnLCVv/zmpv2mjxj336owaZ78412nJO16cxGaHe813jk8nzbfMPPAacxx4Td4mOk2roo2qP1pJ6kuEuIMPAAAABIQJPgAAABCQxC7RAQAAQGBa5C//ikOS+hIh7uADAAAAAWGCDwAAAASEJToAAACIB0t0YsEdfAAAACAgTPABAACAgCR3ic50SUU5Xn/T3I4T3JM2apyACie0x92W86NXvdmeGwqWj/trLCd0KaJwMTtwyBmDqIKgJC/8xtnHbuCQM57O+3NCntwxaDbr8onyvEobNXVGzXCjRvLCi5x+u+ees28qjJo9EbbncI67WnNbQ40aZ7/Ume05nPfnXDPcbTnvr9qocUOenGPqZaPG+eyQvH3j1Owyapzz0+V8Djl9knKHmWXMbXQWgq5iwR18AAAAICBM8AEAAADD008/rUsvvVTl5eUqKCjQypUr8/6btWvX6uyzz1ZRUZGGDRumJUuWdHg/meADAAAgHi0J/GqHxsZGjRkzRosWLbLqd+zYoUsuuUSf//zntWXLFs2aNUt///d/ryeffLJ9DbdTctfgAwAAAAkyZcoUTZkyxa5fvHixhgwZovvvv1+SNGrUKP3hD3/QggULNHny5I7qJnfwAQAAcHxraGho89XU1BTJdtevX6+qqqo235s8ebLWr18fyfYPhwk+AAAA4tGawC9JFRUVKi4uzn7Nnz8/krdbU1Oj0tLSNt8rLS1VQ0OD/vrXv0bSxqGwRAcAAADHterqaqVSqez/FxXlelZ78jHBBwAAwHEtlUq1meBHpaysTLW1bYM8amtrlUql1KNHj8jb+0hiJ/gL/il3nscIcztOboaTYzHBqNlg1EjSGWZdPlHmTjl5NXVme1HlNznb6WPUSF7mkpOj42RvSV4WTa9cQSUfcQKsTM1GmEdhubEhNwTJ2TnO+yvNXyLJC4hJGzXOzqsxaiRpsFHjjMF2sz03KCifA2adcww74T7OBajEqHE54+SOpfP+nEC+QWZ7znnlfDg44VTuce70yTmvnOuP5L0/Z/85N2idcZK8UDAntM8N0Ut99vCvNXwgFT9jbqgTtKrdT67pUB0cdFVZWanf/va3bb63evVqVVZWdmi7rMEHAAAADPv27dOWLVu0ZcsWSR8+BnPLli3auXOnJGnOnDm69tprs/Vf/vKXtX37dt1xxx16+eWX9eCDD+oXv/iFbrvttg7tJxN8AAAAwLBp0yadddZZOuussyRJs2fP1llnnaW5c+dKkt56663sZF+ShgwZot/85jdavXq1xowZo/vvv18//vGPO/QRmVKCl+gAAAAgMEcQLtWh2tmXSZMmKZPJHPb1Q6XUTpo0SX/84x/b2bGjwx18AAAAICBM8AEAAICAsEQHAAAA8fhEuFQiJKkvEeIOPgAAABAQJvgAAABAQI7ZJTpuBomTn+L8lOMEJbn5Z84fbDvbajDbczhZLW7mkpPlUWzUOL81c39CdXJmnIwV94/tnfG0woScICFJaspfUugEzTgHunvyOYFYzsHiBMhIXr+c9pyd7GxH8k4a52RwLmSS9LpR4/TdPe6iCvdxjk33OHDGvMyoaTTbc/aNk5BnXTTkHZ/OtpzrjzNOkne8uOFpDmcMnIA8p0/1Rk2U7bnnnt7I8VrC15wc40/ROVZwBx8AAAAICBN8AAAAICDH7BIdAAAAHGNYohML7uADAAAAAWGCDwAAAASEJToAAACIB0FXseAOPgAAABAQJvgAAABAQFiiAwAAgHjwFJ1YJHaCf9uDUqpHjoJac0NR/Y7i3fwlA93UvzqjZm/+kjL3vUWUWNjHTSJ0EyDzcZIrzRPzU2mjyBlPY7/YdU56Z0mE7RnHsCXKxE0nsvlVsz3n/IsqTdPZd5K03aiJMuXUSY3tY9REGZPtnFfOGlj3uuIm3kbVnpEiHTtnzJ1zwT3unDTfOqPGjUt39rGTEO1cE91rhnG9azSOFfcjZk+Oi8s+cxsIG0t0AAAAgIAk9g4+AAAAApNRsp5ck+nsDnQM7uADAAAAAWGCDwAAAASEJToAAACIB0/RiQV38AEAAICAMMEHAAAAAsISHQAAAMSDJTqxSO4E/zHl7l2puR0nEMMJ1+hp1Dxh1EjSMKOm2qipMNurN2peN2qazfaiCn6pMWqckCDJC2xxEkbcC4ETuuT06SWzvbRZl0+UYxBVe65eRs3bRo0TLuYEWEneOeP8HtUN9jvNqHGOKff66lwXnTAhJzjM2b+SF7rkXFvc66vzGeOMk3suOGFmziMInWPTHXPnvBpu1Owx23PC2l4xapx9nDZqJOt63svYd73MYK2yHHUNH0ha620H4WKJDgAAABCQ5N7BBwAAQFhalaygqyT1JULcwQcAAAACwgQfAAAACAhLdAAAABAPnqITC+7gAwAAAAFhgg8AAAAEhCU6AAAAiAdLdGKR2An+vRukohyvn2Fup9iocXJYxhk1TlaUJPX+j/w1TtZO67Nee07GipMN42RvSV4uiJNP5ZxzTj6O5OUSlRs1TqaNJA0yapw8EycLy1XgHAjOoJvhNxnjxCo42dhQodeeFV7kDOhrRo1zwkjeTnaC4Zx9J0nPGDXOge6+vzqjxjlenPfnnnyNRo3zAWIGDlmhS85xN9Jsz+G053zIuMeBE77lBEqmzfacfXy7UeMcm06/Je+i74znQGfHSDkTMxtapOLN5nYQKpboAAAAAAFJ7B18AAAABIagq1hwBx8AAAAICBN8AAAAICAs0QEAAEA8WpWsJ9ewRAcAAABA0jHBBwAAAALCEh0AAADEg6foxII7+AAAAEBAEnsH/xuDpFSuHz/cCFMnBbPOqDHC5T7l/hTopPCljRr3j1ScBEgj8vcUszkrmdP50dIZTyeqWNKZzrac/eLuYyNJstBJSHSPcyd1M8rkSkNBg1HkvD93DGrNunycY2qvua10RDXuue4kitYZNc65IHlJoLkiyT/ytlHjBnw6ybJ15rYczpg74+kmpjrHnjNWTvKze+69ZNSMMGrctGLnfPiDUePsO/fcc8aq3qgZakSAS1Jjjih75zMYwUvsBB8AAACBaVGynqKTpL5EiCU6AAAAQECY4AMAAAABYYkOAAAA4sESnVhwBx8AAAAICBN8AAAAICAs0QEAAEA8CLqKBXfwAQAAgIAk9w7++8r944cTsKI82/iIEUpkBQC54TdO353QE6ffkhfm4YR0uD/lOmElzn5xgpLcMXDqogrDkqQKo8bZL27wVFShUs5x5/bJCVtJmduKihNi5RwHUQaQNRs1bgCQs2+iqpG8ECtnHzuBhO7tKCc0y3l/7rXF2cdOwJEzlm57zvHivD+nLSnaz0dHVHdc3fPK4YyBc311P2NytRdhYCGOXe26gz9//nydc8456t27t/r166epU6dq69atbWr279+vGTNmqE+fPjrxxBM1bdo01dZGFTEJAACAY1ZLAr8C1K4J/rp16zRjxgxt2LBBq1ev1oEDB3TRRRepsfHjHzlvu+02Pf7441q+fLnWrVunXbt26fLLL4+84wAAAAAO1q4lOqtWrWrz/0uWLFG/fv20efNmffazn1V9fb0eeeQRLV26VBdccIEk6dFHH9WoUaO0YcMGnXfeedH1HAAAAMBBjmoNfn39hwsLS0pKJEmbN2/WgQMHVFVVla0ZOXKkBg4cqPXr1x9ygt/U1KSmpo8X6zY0OIuJAQAAcMxJ2rKYJPUlQkf8FJ3W1lbNmjVL559/vk4//XRJUk1NjQoLC5VOp9vUlpaWqqam5pDbmT9/voqLi7NfFRXOXycCAAAAOJQjnuDPmDFDL774opYtW3ZUHZgzZ47q6+uzX9XV1Ue1PQAAAOB4dkRLdGbOnKlf//rXevrppzVgwIDs98vKytTc3Ky6uro2d/Fra2tVVlZ2yG0VFRWpqMh9PhgAAACOWRklK1wq09kd6BjtuoOfyWQ0c+ZMrVixQk899ZSGDBnS5vWxY8eqW7duWrNmTfZ7W7du1c6dO1VZWRlNjwEAAAAcVrvu4M+YMUNLly7VL3/5S/Xu3Tu7rr64uFg9evRQcXGxbrjhBs2ePVslJSVKpVK65ZZbVFlZ2f4n6JyRp3dOYITkBaj0NbeVT4lZ5/zk6gTyuMEhUQWauD9xlxs1ThDHUKMmyjFIGzXuH+NEFTTi/gjex6hxAlScACc3/MY5DpxtuYFDUZ3HbnuOKIPDHM4+dv7MyQ3bcUKlnKArZ8x3GTWSFyDnnFdue84+dq6dUYb2OX1y9p17vXOuG06olLPvJOldo8b5/HBWBLvnp7MtZzzfNNvLFZj5gbkNBK1dE/yHHnpIkjRp0qQ233/00Ud13XXXSZIWLFigLl26aNq0aWpqatLkyZP14IMPRtJZAAAAHMN4ik4s2jXBz2TyL1Tq3r27Fi1apEWLFh1xpwAAAAAcmSN+ig4AAACA5DmqoCsAAADA1qpkPUUnSX2JEHfwAQAAgIAwwQcAAAACwhIdAAAAxIOn6MSCO/gAAABAQJjgAwAAAAFJ7BKdHz0l9cjx+hnmdpzQPycM1WnPCbKTvMDNKH/yckIGc4XifcQJPpS8Me9p1DQZNW54qdN3JzzYDVA+ydmBUSaYOgPqtOf0233igDNYTnKlO07OgRdVTb1RI3n7JW1uKyovGTWDzG05KbXOyVdk1Lgnu3PcOQnnw832okoKd5Kf3W05nH47CdmS1Hw0HfkE58NK8vp+SkTbcc5hyTs+nePuTPfTP334lxoyUvF75nY6AUt0YsEdfAAAACAgTPABAACAgCR2iQ4AAAACQ9BVLLiDDwAAAASECT4AAAAQEJboAAAAIB48RScW3MEHAAAAAsIEHwAAAAhIYpfoXKTcGThu5kkvI8hijxEc0scIISnelb9Gkno4oS5OmEmJ157eN2qMMI9ebsqTE0zk/ErM+ct2cwzKnPFszF/Sw0nfkqQBRo3z/twxdwJinB/nneAit09OiIxzHDgpbJLXr3eNmqj2neQFeTn7zknjk7ywpD0RtucE9zjXO+ca5R4HzjHlBDO5t7+c4855f26gm9N3Z6yc467OqJG89+d8aLvXFseLRk2FUeMkIEre/nM+h6rNi0v3HBcz47OsU7UqWctieIoOAAAAgKRjgg8AAAAEJLFLdAAAABAYgq5iwR18AAAAICBM8AEAAICAMMEHAABAPFoS+HUEFi1apMGDB6t79+4aP368nn322cPWLlmyRAUFBW2+und3HmV15JjgAwAAAKaf//znmj17tubNm6fnn39eY8aM0eTJk7V79+7D/ptUKqW33nor+/X66693aB+Z4AMAAACm73//+7rxxht1/fXX67TTTtPixYvVs2dP/eQnPznsvykoKFBZWVn2q7S0tEP7mNin6Aw6SUoV5Chwf7Nh/OqljxHy5PyVdY9BxnYk78eqtFGz3WzPCb+JMnSij1HjhCANN2peMWokL7hnqFHjBJVI3ng623LGQJLeNGqc8Bvn2HTCm5LqNKPGCYJyjhXJ28dOuM8Is70XjJqzjRoztC+ywDrnmHLem+RdO539546B81kU5TnjHC9On5xrhhNkJknOPMW5BjufxZJ3Xjmfx8523M9ZZ6ycAKoas710jteiDAzrCAl9ik5DQ0ObbxcVFamo6OCkvubmZm3evFlz5szJfq9Lly6qqqrS+vXrD9vMvn37NGjQILW2turss8/Wfffdp09/+tPRvIdD4A4+AAAAjmsVFRUqLi7Ofs2fP/+Qde+8845aWloOugNfWlqqmppD/4Q2YsQI/eQnP9Evf/lL/cu//ItaW1s1YcIEvfHGG5G/j48k9g4+AAAAEIfq6mqlUqns/x/q7v2RqqysVGVlZfb/J0yYoFGjRumHP/yh7r333sja+SQm+AAAAIjHUTy5pkP8V19SqVSbCf7hnHzyyeratatqa2vbfL+2tlZlZWVWk926ddNZZ52lbdu2tbu7LpboAAAAAIbCwkKNHTtWa9asyX6vtbVVa9asaXOXPpeWlha98MIL6t+/f0d1kzv4AAAAgGv27NmaPn26xo0bp3PPPVcLFy5UY2Ojrr/+eknStddeq1NOOSW7jv+ee+7Reeedp2HDhqmurk7f+9739Prrr+vv//7vO6yPTPABAAAQj4Qu0WmPK6+8Um+//bbmzp2rmpoanXnmmVq1alX2D2937typLl0+XiTz3nvv6cYbb1RNTY1OOukkjR07Vs8884xOO815tNuRYYIPAAAAtMPMmTM1c+bMQ762du3aNv+/YMECLViwIIZefYw1+AAAAEBAEnsH/6vv5c6NOMPcTolRU23UXGbUrDNqJC93qq9R42aQNNfnr+lqbOdts73e7+Svcd5f+sX8Ne5v1pyskhIjNbrJbK/YqHHGvLsxBpI3Dk7WTg+n426IivMGnaCZAWZ7TjCRE67ihPY4Fw0puhN5q9meM55ugJMjqtCl942aU4wayTvQnX47Hx6Sd5w7wX5ueKPz/py+OysD3HPdCTd0njjonC+SZHymWRfhU40a9zZo2qhxjoP+7pKN8Yd/qaFZ+vbPzO10goQGXYWGO/gAAABAQJjgAwAAAAFJ7BIdAAAABKZVyXqKDkt0AAAAACQdE3wAAAAgICzRAQAAQDx4ik4suIMPAAAABIQJPgAAABAQlugAAAAgHi1K1lN0ktSXCCV2gv9PP5NSPXMUuLGqzo5zEgSN9MdT3RS+vUZNnVHjJChK3hg4aZpOMqAkvWvUOGPgMH8H1S9tFDmpjW6yo8Pdfw5nPJ32mo0aN3HTOUedc8ZJZ5W8MXCOFyeu2D0XnMRNZ8yjPNedcXKSZSUvVTWq3xM7SaiSd4464+Qed1FdE9xrotOvXJ+d7dlOnVEjeQmtznnljqVzTBnbatkXyWY+3JZR43w0Nuolq713c9Q1WltA6FiiAwAAAAQksXfwAQAAEBiW6MSCO/gAAABAQJjgAwAAAAFhiQ4AAADiQdBVLLiDDwAAAASECT4AAAAQEJboAAAAIB48RScWyZ3g3yypIMfrTrCG5K2tcsJanHAfNwCoIaJt1ZrtuQFc+dSZdeVGjRPqUmHU1Bk1bl2ZUeMGADmcBBX3mHLOh6gCctw+Oe/PCcNyA4ecMCjnd5bONeNlo0byri1OyFyJ2Z4RyBfptcy5BpUaNU7Yl3PdlKSRZl0+bpiiw7kGu59pzj52jhfnvHKvd4Mias8dA+fz44z8JV2NiV0vZ7wlq+8p5/31MtvLca1u2C/pXnM7CBZLdAAAAICAJPcOPgAAAMLCEp1YcAcfAAAACAgTfAAAACAgLNEBAABAPDJKVrhUprM70DG4gw8AAAAEhAk+AAAAEBCW6AAAACAePEUnFomd4H+pIXduxHhzO062Rsu7+WsmGtt53qiRpOFGjZOPU222d4rx/px8o61mexVG+I3z/oZvy1/jZtG8atQM3pG/xs3/qTNqnDFwsrckPwsqb3vGgdBiht90LTaKnAurO+jGcW4F8hjb+Wu9sR1JPYYZRc4YNHrtWRcFIwDI/sAbYNQ4Y+WcDE64keT13WnPCeiSvJMvqsAzyTsfnGAtJ1DJCSCTpNOMGudifbbZnnMsDDVqnHFyAvQkbz3EiU4K5OfMBq85/EsN70v3fsncDkLFEh0AAAAgIIm9gw8AAIDAtCpZT9FJUl8ixB18AAAAICBM8AEAAICAsEQHAAAA8eApOrHgDj4AAAAQECb4AAAAQEBYogMAAIB4sEQnFtzBBwAAAAKS2Dv4i5Q7/K+fE70q5Y7D/S8ZI2mx4PT8NRdvz18jyUsidH706mO2V2fUGOM5yU0UNZJsnf2i/flLygYb25F0hpOi6PRpj9ee0hHVuAmmDie10XgecFf3OHjFqHGiep3wR0lyzj/nvBqTv6SH+9xkpz0nwdRNFHVSancZNU7yquTd+XKOOyeZ1ElLlbxnWhsp2Soy23MSYZ32nOuP5H1+OGnTzj42rsGSont//2G253COTWffuXd3K4yavcbJN/xfvfa65aj7q7cJhC2xE3wAAAAEhqCrWLBEBwAAAAgIE3wAAAAgICzRAQAAQDx4ik4suIMPAAAABIQJPgAAABAQlugAAAAgHq1K1rIYnqIDAAAAIOkSewe/71VSKlcATI25oa75Swqc4BDnp00nZEbygqec9pztSFKpUeOE0bihJ844OAFOTpBXnVEjScPMunwOmHVOsJZjkFnn3IFwjik34MjhjLkTyOOGfY0wapz3V2fURHnHx9kvQ81tOSFzzjHlHudO3btGjXNtcUOQnPA0J8TK3cfO+3OOO/ea4YSCRXWcRxns53DH3DlenM/1EqPG3S9/MGrSEW0nnw8i2AaOee26g//QQw9p9OjRSqVSSqVSqqys1BNPPJF9ff/+/ZoxY4b69OmjE088UdOmTVNtrfOJAwAAgOC1JvArQO2a4A8YMEDf/va3tXnzZm3atEkXXHCBLrvsMv3nf/6nJOm2227T448/ruXLl2vdunXatWuXLr/88g7pOAAAAICDtWuJzqWXXtrm/7/1rW/poYce0oYNGzRgwAA98sgjWrp0qS644AJJ0qOPPqpRo0Zpw4YNOu+886LrNQAAAIBDOuI1+C0tLVq+fLkaGxtVWVmpzZs368CBA6qqqsrWjBw5UgMHDtT69esPO8FvampSU1NT9v8bGhqOtEsAAABIMoKuYtHup+i88MILOvHEE1VUVKQvf/nLWrFihU477TTV1NSosLBQ6XS6TX1paalqag7/F7Hz589XcXFx9quioqLdbwIAAADAh9o9wR8xYoS2bNmijRs36uabb9b06dP10ksvHXEH5syZo/r6+uxXdXX1EW8LAAAAON61e4lOYWGhhg378Pl3Y8eO1XPPPacf/OAHuvLKK9Xc3Ky6uro2d/Fra2tVVnb455YVFRWpqMh5ZhkAAACOaUl7ck2S+hKhow66am1tVVNTk8aOHatu3bppzZo12de2bt2qnTt3qrKy8mibAQAAAGBo1x38OXPmaMqUKRo4cKD27t2rpUuXau3atXryySdVXFysG264QbNnz1ZJSYlSqZRuueUWVVZWHtkTdJ5X7pAqd6m+E1Lh/Jjj/ITnhsN0M2qckA6XEyYUZVyBsy0niMUJdHHHvM6ocQJynO1IXkhXs1Hjvj8n+MUJM3P2XbFRI0mvGzVRBmsZoXbWueCMpXMOu9JGjTOWkvfHYrkCBD/iHJuS1MuocX5B6xybbuiS85yGlFHj/uGdcwxHdX66nP3XlL9Ep5rt1Zl1+aTNOjd0MR/n+uqe62fH3F6ueUuzpA3mdhCsdk3wd+/erWuvvVZvvfWWiouLNXr0aD355JP6whe+IElasGCBunTpomnTpqmpqUmTJ0/Wgw8+2CEdBwAAwDGGp+jEol0T/EceeSTn6927d9eiRYu0aNGio+oUAAAAgCNz1GvwAQAAACTHEQddAQAAAO3CEp1YcAcfAAAACAgTfAAAACAgLNEBAABAPAi6igV38AEAAICAJPYOftlfpIIcr0/+s7ed4UbNM0bNV4yafzFqJOkco8bJPHHemyTVGDVOtsZ2s720UePklDkZZUONGkn6D6Omr3FMNZrtTTRqnBwvN3/sNKMmquw0dwyc9py8msFme12NQKWMEe5TUJq/ptHcMT2NmoJRRpF78jkBawMi2o7khU8523IOhLRRI3mhS86OcYLT3G05J7sb3lhu1Dh/NOhcPN1AKSfAyXl/I8z2thk1zkXRONd1QolRJEkXGjVOmuIlZnufO/xLDQ3SUufNIWSJneADAAAgMK1K1pNrWKIDAAAAIOmY4AMAAAABYYkOAAAA4tGiZN1eTtJyoQglaYgBAAAAHCUm+AAAAEBAWKIDAACAeBB0FQvu4AMAAAABYYIPAAAABCSxS3SelnRijtedMD/JS9N816gpK85fM7ne2JCkXl5ZXl1zDdAnOSmDTmqjm2robMtIHbV+beakSEr6jLOTnX67PxKnjRonPvh9sz1nHJz2Co0adwyi2n9uBK+RFlrgPC3BOA56uX1ywiSdcXIjm08xapz95ySvSl4wp3MMO31yjzs3hTcf53yRvLGqM2peM9tzosmd49xJY24waiTvQ82JjB9stueMwb8ZNYOMmq7Oh4ek3svz16SN7XT7Z6+9XKm/+7xNdBqeohOLJA0xAAAAgKPEBB8AAAAISGKX6AAAACAwPEUnFtzBBwAAAALCBB8AAAAICEt0AAAAEA+eohOLJA0xAAAAgKPEBB8AAAAISGKX6HzqO1KqR44CN3SpKX9JmRO69Eb+kpQRhiVJetuocUKX3CAWhxPW0tvcltN351diaaPGCTyRvLAv5/054TCSl8S2x6hxxlLywoSc0KXXI9qO5B3nbmBUVJzj3DlWnIAcyUvRc69ljigD6xzOtpzbSM6YO8eTK6q0Qcl7f06AnLtfnGPK+Wx4yWzP4Vw7nWvUOrM953hx2jP6nXGuiZIKSoyi5vwlDWZIVa635+bUdRqW6MQiSUMMAAAAJN6iRYs0ePBgde/eXePHj9ezzz6bs3758uUaOXKkunfvrjPOOEO//e1vO7R/TPABAAAA089//nPNnj1b8+bN0/PPP68xY8Zo8uTJ2r179yHrn3nmGV199dW64YYb9Mc//lFTp07V1KlT9eKLL3ZYH5ngAwAAIB4ZfRx2lYSvTPvfwve//33deOONuv7663Xaaadp8eLF6tmzp37yk58csv4HP/iBLr74Yv3DP/yDRo0apXvvvVdnn322/uf//J/tb9zEBB8AAAAwNDc3a/Pmzaqqqsp+r0uXLqqqqtL69esP+W/Wr1/fpl6SJk+efNj6KCT2j2wBAACAODQ0NLT5/6KiIhUVHfwUlnfeeUctLS0qLW37xInS0lK9/PLLh9x2TU3NIetratwnhbQfd/ABAAAQj5YEfkmqqKhQcXFx9mv+/PkdNwYx4A4+AAAAjmvV1dVKpVLZ/z/U3XtJOvnkk9W1a1fV1rZ9bnZtba3KysoO+W/KysraVR8F7uADAADguJZKpdp8HW6CX1hYqLFjx2rNmjXZ77W2tmrNmjWqrKw85L+prKxsUy9Jq1evPmx9FBJ7B3/u16Rc+VPDzO30MWrqjJrPGTW/M2rcbTUaNU6Oh+RlkDg12832nIwnJ4cllb/E/gm1zqhxsm/cLBpnDJzsmyjzN5y+p3oaRU7IjGQFYjUbAXKFTp8k6XSjxgn3GW7UPG/USJLTd+fAcw8EJ+iqr7ktxy6jxgkzc/rk9ttpz7nguce5s4TWCUFyQ9+ci17aqBlj1Ljhhs4x7ARBumPuHOdOIJ8xQSioMLYjeR8yxvUgNcr55JNSGnn4ZhpapOLN1nY6RYukgs7uxCccwQft7NmzNX36dI0bN07nnnuuFi5cqMbGRl1//fWSpGuvvVannHJKdpnPrbfeqs997nO6//77dckll2jZsmXatGmTHn744SjfSRuJneADAAAASXPllVfq7bff1ty5c1VTU6MzzzxTq1atyv4h7c6dO9Wly8c/jU+YMEFLly7VnXfeqX/8x3/U8OHDtXLlSp1+unNn6sgwwQcAAADaYebMmZo5c+YhX1u7du1B37viiit0xRVXdHCvPsYEHwAAAPH4KGAqKZLUlwjxR7YAAABAQJjgAwAAAAFhiQ4AAADiEcBTdI4F3MEHAAAAAsIEHwAAAAgIS3QAAAAQD56iE4vETvArlTv0zQ2Xc8IP3zZqTjV+1/G+eZA4fXfCEQd7zUUWfOiks0pewq6TZOvU9HCKJPUxEhKdZXjuUr0eToqi0feubrKjUVcY1aC7iZtGdG6hE0ntpFZKUpNR44REOuexk0wqeXHFDvc4cC54TqSxG5PtXMz2GjXOieXEkkteoqizj90E5bRR865R41wzXM7x4pzrzoejJNVHtK0Iry2qNmqcD7XXjBrJS/N1tDZ4daXPHv4155xD8FiiAwAAAAQksXfwAQAAEBieohML7uADAAAAAWGCDwAAAASEJToAAACIR6uStSwm0KfocAcfAAAACAgTfAAAACAgLNEBAABAPFqVrKfoBLpEJ7ET/CklUirX7xfcIIfB+UtOcbZjhJ6c4YbDGNs61QkFiTL8xggT6mHmb/RwwoScoJJmo8YMRulqhJBYoVLuukEnCGm7URNl2I4TGOX8Ti/KMJq0uS1Hb6NmkFFTZ9REeK5b+8U9152wHefaUmy259hj1Djnuhsk5IQXOZ8f7j52jnPnGuxOMpzQLOf9OWPuco5PJ6isyGzvDaNmgFFTZ9S4+yWqRMn/iKA9J/QPwWOJDgAAABCQxN7BBwAAQGCS9AQdKXn9iQh38AEAAICAMMEHAAAAAsISHQAAAMQjaUtiktafiHAHHwAAAAgIE3wAAAAgICzRAQAAQDwIuopFcif4pcodAFNobsf5HYUTaOKE+7i/D3GCUboZNUPN9pxgm7RRE+U6NSdIx9kvTkiQ5AWxOPvP3cdOGI0T/NJotueE+zjBWs54Ou9Nks4waqqNmjqzvTFGzStGTYVRU2/USNFdW9xgP+e8cq4b7rnuXMucADInxMo9F6IK8nL3sXNNeM2oKTPbc0L0dkW0HTdczDkOnP0X5XFXa9Q4QXTO9UDyrlPOueBeX3ONQZQhZjhmsUQHAAAACEhy7+ADAAAgLEl7ak3S+hMR7uADAAAAAWGCDwAAAASEJToAAACIB0/RiQV38AEAAICAMMEHAAAAAsISHQAAAMQjaUtiktafiHAHHwAAAAhIcu/gb+gtpXL8FcafGrztOAmQg42adUbNRKNGkl4w6/JxkvokLz2vzqhx0lJdTt/7GjVOEqrk9d1JR3STHV8y6/JxkhYlLzHVSZZ13p/7zGAnobXSqHGSSSXvXL/cqKkxak4zaiQvQdlJrnSPA6c9h5OEKnn9cs7jJqPGTfN1knqdMXdvf7n9yse9ZjiJsM4dyT35S1qc/SKpq5GK22KMuTvkzmHu1Lxp1LiBxq8ZNU5QeBSHgbnbELjkTvABAAAQlhZJmc7uxCewRAcAAABA0jHBBwAAAALCEh0AAADEI2lLYpLWn4hwBx8AAAAICBN8AAAAICAs0QEAAEA8eIpOLLiDDwAAAATkqO7gf/vb39acOXN06623auHChZKk/fv36/bbb9eyZcvU1NSkyZMn68EHH1RpaWk7t16knD9/VJhBV05IjhPW4tS4gSenGDVO2oUTIONywmGclA7JSxjpZtS8ZrbncEKXnP1XZ7Y3yKhxwqmc8C3JC/JyjmEnOKzQqHE5x5QbdDXSqHH2sRP2VWfUSNElzjjBTJJUYdQ415Yisz3nvHKCw5xjyg2Zc45h5zhwr3dRBuQ5nLuNzUaNcZ3u6lynJTUax2eUl3xnOKM6DFzOqeAcUu4l//Ucr0WVd4dj2xHfwX/uuef0wx/+UKNHj27z/dtuu02PP/64li9frnXr1mnXrl26/HInPhIAAABBa0ngV4COaIK/b98+XXPNNfrRj36kk046Kfv9+vp6PfLII/r+97+vCy64QGPHjtWjjz6qZ555Rhs2bIis0wAAAAAO7Ygm+DNmzNAll1yiqqqqNt/fvHmzDhw40Ob7I0eO1MCBA7V+/fqj6ykAAACAvNq9Bn/ZsmV6/vnn9dxzzx30Wk1NjQoLC5VOp9t8v7S0VDU1h16I2dTUpKamjxeqNjSYa+sBAABwbEnaU2uS1p+ItOsOfnV1tW699Vb97Gc/U/fuzp+U5Dd//nwVFxdnvyoqnL8SAwAAAHAo7Zrgb968Wbt379bZZ5+tE044QSeccILWrVunBx54QCeccIJKS0vV3Nysurq6Nv+utrZWZWVlh9zmnDlzVF9fn/2qrnb+9h0AAADAobRric6FF16oF154oc33rr/+eo0cOVJf+9rXVFFRoW7dumnNmjWaNm2aJGnr1q3auXOnKisrD7nNoqIiFRW5z2QDAADAMatVyQq6SlJfItSuCX7v3r11+umnt/ler1691KdPn+z3b7jhBs2ePVslJSVKpVK65ZZbVFlZqfPOOy+6XgMAAAA4pKMKujqUBQsWqEuXLpo2bVqboKt2+/o7ucNWtpvbSRs1UYVYucEozjNXnbQLtz0zrCQv9w9RnJAVh5Nmssfc1iajxtnH7lg6+885Dtx97NQ1GjXO+3OPg95GTZRj8H+NmpRR4xxTaaNG8o4D57hzk2uc9kqMGudYkbx+RfUHbG4CkPP+nD8hc4LoJP9YyMf5HJK8ED1nrJz3V2fUSOo12CgyzqtTD72S92C78peUOdefOqPGvf441w3jTwy/+IrZXp/Dv9TQIv3bNnM7CNZRT/DXrl3b5v+7d++uRYsWadGiRUe7aQAAAISkVVJBZ3fiEwJdonPESbYAAAAAkocJPgAAABCQyNfgAwAAAIfUIpboxIA7+AAAAEBAmOADAAAAAWGJDgAAAOLBEp1YcAcfAAAACEhi7+D/9aHcmTs9nDATV51R47TnBqM4ISRDjRonrEWyw0ryijJsxwj8sNpzw2+qjZozjJoasz0nZMXh7mNnHJzgMGc7UYZ9OX1ywrDcOueYct5flOd634i2I3nH+XCjpslsz6lz3p8z5u6597pRU25uy+Gc61EdB1J0x7lzDXaDp5zrlBNO6e6XUqPGGU8nxMo995yxco5z9/qaq+9/lfRVczsIVmIn+AAAAAgMQVexYIkOAAAAEBAm+AAAAEBAWKIDAACAePAUnVhwBx8AAAAICBN8AAAAICAs0QEAAEA8WKITC+7gAwAAAAFhgg8AAAAEJLFLdHqkpB65foUTRdrbR5zUWEfarHP6Xm/UOKmVkpcOmDJqGsz2nGTHRqMmbdS4x8EIsy4fN2nR6ZdzbLqJuHuNGnes8nFSJCUvQdlJm3zbbM9N/c3HSa502+pp1Owyak4323vXqGmOqEbyzlHnOuUcB4VGjeSNedqoiep4kqJLUJa888oZg6jakrxzxrmd+IrZnjNWTsqycy1zrq1SdJ/rJWZ7uT4b3PTdzpJRsMtikoQ7+AAAAEBAmOADAAAAAUnsEh0AAACEpeW/vpIiSX2JEnfwAQAAgIAwwQcAAAACwhIdAAAAxIIlOvHgDj4AAAAQECb4AAAAQEASu0Tnhw1Sjxyvn+4ERsjLF3EyQU41arYYNZJ0mlHj9Pt1s72hO/LXOJkuNWZ7TjaTU9OvKH9NoxNmIi8raZBRU+c1Z+XoOPkpqWKvvYxxPhScaGyol1HjhJRJXtjKG0aNexuij1Gzx6hxTobtRo0kDTdqnICcWrM95/1FFcImecFazsXM+R25G77lhtHl4wZdOf1yg8oczvlXYdQ4F8Uyo8blXMtaI2zPCbFyxsnlXO+cQLeRZnupHOmUDRnpXjehK36tinZXH60k9SVK3MEHAAAAAsIEHwAAAAhIYpfoAAAAICw8RSce3MEHAAAAAsIEHwAAAIjYu+++q2uuuUapVErpdFo33HCD9u3bl/PfTJo0SQUFBW2+vvzlL7e7bZboAAAAIBbH01N0rrnmGr311ltavXq1Dhw4oOuvv1433XSTli5dmvPf3Xjjjbrnnnuy/9+zp/O8x7aY4AMAAAAR+vOf/6xVq1bpueee07hx4yRJ//zP/6wvfvGL+qd/+ieVlx/+mb49e/ZUWdnRPaeWJToAAABAhNavX690Op2d3EtSVVWVunTpoo0bN+b8tz/72c908skn6/TTT9ecOXP0/vvvt7v9xN7B/3+HSalcYSvubyucIBZnW0bwy8VOgIzkBYwYaVinOu9N8gI4jDCaMvdPzZ1xcAI/Tslf0utVYzuSejmBSsPyl5xU6LWnN/OX5Igp+ZgT3iSpwNnHzo/zzvtzj/OoAo6c8C3JSyEbbNQ4fXLDlJxENydQydm/kne9Gx/RdiSvXwPMbeXjhn05v28vMWqc67TkXcuc8XQD5Jz35wax5fOKWed8hjrbGmq2V23U1Bk1fzRqnPA4yQuHc64H7rneq+Hwr5kBkJ0lqU/RaWhoO6ZFRUUqKjISNw+jpqZG/fr1a/O9E044QSUlJaqpOXx06H//7/9dgwYNUnl5uf70pz/pa1/7mrZu3ar//b//d7vaT+wEHwAAAIhDRUXbaON58+bprrvuOqju61//ur7zne/k3Naf//znI+7HTTfdlP3vM844Q/3799eFF16oV199Vaeeeqq9HSb4AAAAOK5VV1crlfr4d+uHu3t/++2367rrrsu5raFDh6qsrEy7d+9u8/0PPvhA7777brvW148f/+GvXbdt28YEHwAAAMnTqmQt0flo1VsqlWozwT+cvn37qm/f/GvzKisrVVdXp82bN2vs2LGSpKeeekqtra3ZSbtjy5YtkqT+/fvb/0bij2wBAACASI0aNUoXX3yxbrzxRj377LP693//d82cOVNXXXVV9gk6b775pkaOHKlnn31WkvTqq6/q3nvv1ebNm/Xaa6/pV7/6la699lp99rOf1ejRo9vVPhN8AAAAIGI/+9nPNHLkSF144YX64he/qIkTJ+rhhx/Ovn7gwAFt3bo1+5ScwsJC/e53v9NFF12kkSNH6vbbb9e0adP0+OOPt7ttlugAAAAgFsdT0FVJSUnOUKvBgwcrk8lk/7+iokLr1q2LpG3u4AMAAAABYYIPAAAABIQlOgAAAIhFUoOuQpPcCf4+5f79QpS/e3ASgJ30zr1me07i3etGTZRHpdMnN2HP2ZYz5kYarD3mzv6rN2rc485JhG02atwxdzgLDZ1jyum35KXwOvsvR2BjG05KZJ25rXzcpFeHkzrpppw6Cbt15rYcxUaNc647ab5uenlUnw1O2q3kvT83jdnhXBOcsXKSpl3Oeexcf3aZ7TljsNWoKTVqzDRx6zoV5XGea1sJT7JFPFiiAwAAAAQkuXfwAQAAEBSW6MSDO/gAAABAQJjgAwAAAAFhiQ4AAABicTwFXXUm7uADAAAAAWGCDwAAAASEJToAAACIBU/RiUdyJ/iTlDssyP3dw9tGjRP44SzS6mvUSFK1UTPUqHFCMyQvMMoZAycsyuX0PW3UlJntOcFERUaNE2AlecedE0rkbEfyQnmcMXBCytzQpbRRs8eocY9zhxNKNMioiTLUzrmWOSFeLid0yf3EqzFqTjVqnJAgt0/Otcw5j6M8914xatJme04QkhPa52zHPdedMXdq3DF3zgcnDKvOqHGvP862nM9Q5xol5X5/oc5Y0S4s0QEAAAACktw7+AAAAAgKT9GJB3fwAQAAgIAwwQcAAAACwhIdAAAAxKJVyfo7YJboAAAAAEg8JvgAAABAQFiiAwAAgFgQdBWPxE7w/79luXM4nCwayct0cbI1xhs1rxk1kten7n/IX1NqtufkZjgHuJNT4nKyQ5yMlX5OOJWk95ry1zjr8NzMk15Ov4yNNTuBNZIKndAsJ0zICQByr4bbjRon+MU5YSSp2ah53ahxfq/pBvKcYtbl4wYOOaFLTgCQO+ZOndN34/zUaUaN5O0bp99OEJS7LSfwbITZnsMJYhtu1DjHiuTtY+uDz2zPuZaljZoKo8YNsHT67tQ4IZdS7nO9QVIfczsIFkt0AAAAgIAk9g4+AAAAwkLQVTy4gw8AAAAEhAk+AAAAEBCW6AAAACAWPEUnHtzBBwAAAALCBB8AAAAICEt0AAAAEAuW6MSDO/gAAABAQBJ7B39auZTK9eOHmy7nJvHlk85f8umo2pK82Fgz9c8KpXR+1HPSJiUvndSo6VVnbKe3USPpJCfZ0Rko90di58G6xj4udG8tOOeDk6bpHMPO/pW8RFEnedVJ15W82zATjJpaoyZt1EjemDvj6YyT294eo8Y8r6y0V+c4cMZgl1HjbiuqY1OS6owaZzyd5GfJG3MnvtxJn3Xjy6O6trhj7vTLSa1+yahxPjskabBR4yScu2Oe6/Pf/axG0BI7wQcAAEBYCLqKB0t0AAAAgIAwwQcAAAACwhIdAAAAxIKn6MSDO/gAAABAQJjgAwAAAAFhiQ4AAABikVGynlyT6ewOdBDu4AMAAAABadcd/Lvuukt33313m++NGDFCL7/8siRp//79uv3227Vs2TI1NTVp8uTJevDBB1VaWtr+nn1NUo8cr5shT6oxapyQDjfswuGEEjn9dsNonNCL5pjbqzNqnECePkaN5AWoOEE6TsiM5AXNlBk1ThiNK6rAoXfN9sqNGifUJe4AuQFGjRsk02DUOLey3HAx51rm/EWZlY4n75gaatQ411fnvUnetcW5vrp/eef03dnHzrEi+fsmH+dccD4XJO96lzZqnjTbO8OoMUKsGo0x6OVe843r1M4Ib1vnGvJ90TWDY1i7l+h8+tOf1u9+97uPN3DCx5u47bbb9Jvf/EbLly9XcXGxZs6cqcsvv1z//u//Hk1vAQAAcMziKTrxaPcE/4QTTlBZ2cG3Huvr6/XII49o6dKluuCCCyRJjz76qEaNGqUNGzbovPPOO/reAgAAAMip3WvwX3nlFZWXl2vo0KG65pprtHPnTknS5s2bdeDAAVVVVWVrR44cqYEDB2r9+vXR9RgAAADAYbXrDv748eO1ZMkSjRgxQm+99ZbuvvtufeYzn9GLL76ompoaFRYWKp1Ot/k3paWlqqk5/ILHpqYmNTV9vKC1ocFdiAgAAIBjCUt04tGuCf6UKVOy/z169GiNHz9egwYN0i9+8Qv16JHrL2IPb/78+Qf94S4AAACAI3NUj8lMp9P61Kc+pW3btqmsrEzNzc2qq6trU1NbW3vINfsfmTNnjurr67Nf1dXVR9MlAAAA4Lh2VBP8ffv26dVXX1X//v01duxYdevWTWvWrMm+vnXrVu3cuVOVlZWH3UZRUZFSqVSbLwAAAISnNYFfIWrXEp2vfvWruvTSSzVo0CDt2rVL8+bNU9euXXX11VeruLhYN9xwg2bPnq2SkhKlUindcsstqqys5Ak6AAAAQEzaNcF/4403dPXVV2vPnj3q27evJk6cqA0bNqhv3w+TmxYsWKAuXbpo2rRpbYKujkTjrblzTXoNMjdUd0TNHywdYVvOtpxgmyjbc4JK3NAlJ8DJCVApNNtzvGnUDDdq3JCntFHjBDi5AUfOvnEC1pwwIbdPTnBY2qhxQ+2cY8oJRnvNqHGDrpxwOOf9uWO+3aiJ8i/KnGPYOWec0CWXc5xHtV8k7zrlhH25q1PrjBonZC7K25ZOkKBzrIw320sbNRflL+nl7GP3mm+MwUDnc7bCbC/H+ouGv+rDsFAc19o1wV+2bFnO17t3765FixZp0aJFR9UpAAAAhIen6MTjqNbgAwAAAEgWJvgAAABAQNq1RAcAAAA4UizRiQd38AEAAICAMMEHAAAAAsISHQAAAMQiaeFSSepLlLiDDwAAAAQksXfwm5Q7F6PL6952epQaRU6QhRM+4QaVOKEge42atNmewwkhcf8SxQldcgKVnO30MmokL7SmzqhxxkmSiiJqb4/ZnjMOTjiT8/56GjWSdww77bnBLymj5jWjxgnDOsWokaS3jRpnH7shc8VGjXO9c4O16oyaGqPGuR644YYO59h0+iR5+8apcc8r51rmBHk5Y+BeX52gMieAzDlfJKnBqHH6lDZq3OPAGXOHe+6F+pehiExiJ/gAAAAIS6uS9fMJS3QAAAAAJB4TfAAAACAgLNEBAABALHiKTjy4gw8AAAAEhAk+AAAAEBAm+AAAAEBAWIMPAACAWLQoWY/JTFJfosQdfAAAACAgib2DX7JHSuVKpnze3JCTZldu1Gw0aoYaNZKXJOmkDDrvTfJSVeuNGicV2FVr1Dg/Vrtj4KShvmbUuGmFTkLr+0aNc6xIfgpmPnVGjZMYK3n7zznOXemItlNn1LjJss61xdnH7i2mMqPmVaPGOTYlL/XXGSvn/bkpp06f6owa99rinOuvGDVvmO051yA3ATuPzDtenTNUhU6Ce53XXsZ47InzEeMcmu5h5wQ2O4fmOrO9XIedE1qO8CV2gg8AAICwsEQnHizRAQAAAALCBB8AAAAICEt0AAAAEAuSbOPBHXwAAAAgIEzwAQAAgICwRAcAAACx4Ck68eAOPgAAABCQ5N7B3ySpV47X3YAcJ0TGSajYZdR0M2okr+9O0Ex3sz0nqaPZqHF/zI0q5MnptxOeInl9ijJcrM6o6WrUOP2WvGMvqv3ipqg0GDXuMexwUmucMYhyvzjHi9OeG6zlXMuc/ecGpznHS9qoccbJDZlzrlPO+eJcEyVv/zl/xVdstldo1DhjZfS7wElmklT4mlHk3E40rwcFxrZ67/O2lU/arBth1DjTkYlme7m25ebUIWzJneADAAAgKCzRiQdLdAAAAICAMMEHAAAAAsISHQAAAMQio2SFS2U6uwMdhDv4AAAAQECY4AMAAAABYYkOAAAAYsFTdOLBHXwAAAAgIMm9g3+SpBNzvO7+yOWEJQ02arYbNWmjRvLSLpzgFyd8S5LKjRonSMdNz3BCVpy/sBlm1Lhj4ATbOOE+TqiN5AW2OH13A4ec4yVXcNxHnACntFHjcoLK3FA7ZwzKjJr/MGrcULu0UbMnou1I0YV0ue/PGc9XImrPGSfJu54756cbwuZcF51rsBNSJnmfaU7fa40a93PWubZEmb5khLU5Xe9r1Lxm1LjtpYwad5hy3Z3lzi0kjgMAAADEpDWBXx3lW9/6liZMmKCePXsqnU5b/yaTyWju3Lnq37+/evTooaqqKr3yinOnpC0m+AAAAEDEmpubdcUVV+jmm2+2/813v/tdPfDAA1q8eLE2btyoXr16afLkydq/3/n168eSu0QHAAAAOEbdfffdkqQlS5ZY9ZlMRgsXLtSdd96pyy67TJL02GOPqbS0VCtXrtRVV11lt80dfAAAAMSiJYFfktTQ0NDmq6nJ+GOPiO3YsUM1NTWqqqrKfq+4uFjjx4/X+vXr27UtJvgAAAA4rlVUVKi4uDj7NX/+/Nj7UFPz4V/bl5aWtvl+aWlp9jUXE3wAAAAc16qrq1VfX5/9mjNnziHrvv71r6ugoCDn18svvxxz7w/GGnwAAADEIqlBV6lUSqlU/oeZ3n777bruuuty1gwdOvSI+lJW9uGzh2tra9W/f//s92tra3XmmWe2a1tM8AEAAABD37591bevk6LQfkOGDFFZWZnWrFmTndA3NDRo48aN7XoSj8QSHQAAACByO3fu1JYtW7Rz5061tLRoy5Yt2rJli/bt25etGTlypFasWCFJKigo0KxZs/TNb35Tv/rVr/TCCy/o2muvVXl5uaZOndqutpN7B3++cqcbOqmVyrONj/zaqHHSNN0/cHZ+rIoy9e+liLbTHNF2JC9ZIsofPwuNmigTPp0f7qNMK45KnVHjJnw6ovw9bdqoqTZqnFRnJ7lT8lJcnTFwE5Sd49OpccZA8vruJFs7ic1uGo0z5oOMGvcBGnGnVg82apxrp7OCIMoUaWec3OPO2FbKueYbn2mnGpuxGeP5Rfc4z9H3hhZJfzG30wk6OlyqvTqyL3PnztVPf/rT7P+fddZZkqTf//73mjRpkiRp69atqq+vz9bccccdamxs1E033aS6ujpNnDhRq1atUvfu7fvwLchkMpmjfwvRaWhoUHFxser/HykV1wTf4Vzo3LGPe4If1YSMCX78E3z3AzYqdUbNsTzBd8Y8ygl+nVET9wTfGQN3vzjHgtNelBN854e4KCf4Dufa4k7wnWMhqmvnMTzBt675UX6mOZzxjGiCX/wXqb6+3lpTHpeP5nf3S+rR2Z35hL9Kul3JG6+jxRIdAAAAICDJXaIDAACAoCT1KTqh4Q4+AAAAEBAm+AAAAEBAWKIDAACAWLQqWctikvREnyhxBx8AAAAICBN8AAAAICDJXaIzXrmfdes+v9Z5nrzz7OjtRk25USN5z0R2npfrPhvbeWawE0bj9FvyxrzIqHnDqHGfHe08l94Z8yif0+wcd85+kbzfd0b1/HO3T877c541XmO25+zjt42atFET9/P73ewBZx+/adScYrZXZ9Q4fXJ+R+5mnzj7OKpAMElyHptdbNS4z5x3xsr5fHQ+P9xzvTaibT1vtud81jqfQyVmew7nuHOYx8GrOY6DuONT2ut4CrrqTNzBBwAAAALCBB8AAAAISHKX6AAAACAoBF3Fgzv4AAAAQECY4AMAAAABYYkOAAAAYsFTdOLBHXwAAAAgIEzwAQAAgIAkd4lOhXKHGDkhFpIXTOQEOJ1m1LjBKE5gixMANMpszwkhiTt0yQkFcX78dINKnKAgJx0kbbbnhMhEFUokSYONGuc4d8Ki3IA15zhwjrsysz03DCqftFHj7hf3nMnnXbNukFGzxajpY7bnHAvOeeWM5zlGjSSdbdS8ZtS4wYVOIN8wo+Yls71Co8Y5r9JGjdsn5/PYuB5kzLUSBcYx9Wp9/pqhxnnlLt9wLq9OLuMus71cdW4eY2fhKTrx4A4+AAAAEBAm+AAAAEBAkrtEBwAAAEFhiU48uIMPAAAABIQJPgAAABAQlugAAAAgFgRdxYM7+AAAAEBAmOADAAAAAUnuEp2Bkk7M8fpr5nacP492gnReNmqckBkp2kAlhxMG5fyOyknpkKQ6o8YZ85RR46SLSF7fhxo1bnDRpoi25QSsSV5olnPcOWFRbopKL6PGCUpyH3FQY9RE9f4GGzWSdy44x3Cp2d7rRs0ZRo2z7yTv/aUjqnGOX8nbf057buKQc21x9rEb1OZcq50xiGqcJK/vRrhhQZHZnhHkdaqxrd1GoGQ6f4kk71Rw8h2rzfZeyfHaX81tdJZWJevJNSzRAQAAAJB4TPABAACAgCR3iQ4AAACCQtBVPLiDDwAAAASECT4AAAAQEJboAAAAIBYEXcWDO/gAAABAQJjgAwAAAAFhiQ4AAABiwVN04tHuCf6bb76pr33ta3riiSf0/vvva9iwYXr00Uc1btw4SVImk9G8efP0ox/9SHV1dTr//PP10EMPafjw4e1raPRUKZUjru605d52nATBVHn+mnIj1nCM0ZbkxdnVGjXuUWmk/lntOUmvbnu5Yvg+Ume25xhp1DgRgm7apJNS6yZzOpxEWKc955hy9q/kpes6nPNF8tJXneuBMwZuorGTIr3dqHHH3Ek+dsaz2WzP+R2wM55OquopRo3kjfnzRk1fsz3n2ukcd++a7Rnj+dd3zG3l4VxWpOiCerea7fU0Emj/w9iOc4lyP2ad4GNnnJ6LoL1QJ6xon3Yt0Xnvvfd0/vnnq1u3bnriiSf00ksv6f7779dJJ52Urfnud7+rBx54QIsXL9bGjRvVq1cvTZ48Wfv3O4c2AAAAgKPRrjv43/nOd1RRUaFHH300+70hQ4Zk/zuTyWjhwoW68847ddlll0mSHnvsMZWWlmrlypW66qqrIuo2AAAAjjU8RSce7bqD/6tf/Urjxo3TFVdcoX79+umss87Sj370o+zrO3bsUE1NjaqqqrLfKy4u1vjx47V+/froeg0AAADgkNo1wd++fXt2Pf2TTz6pm2++WV/5ylf005/+VJJUU1MjSSotLW3z70pLS7Ov/a2mpiY1NDS0+QIAAABwZNq1RKe1tVXjxo3TfffdJ0k666yz9OKLL2rx4sWaPn36EXVg/vz5uvvuu4/o3wIAAODYwVN04tGuO/j9+/fXaae1fUzDqFGjtHPnTklSWVmZJKm2tu1jBWpra7Ov/a05c+aovr4++1Vd7TzKBAAAAMChtGuCf/7552vr1rYPsvrLX/6iQYMGSfrwD27Lysq0Zs2a7OsNDQ3auHGjKisrD7nNoqIipVKpNl8AAAAAjky7lujcdtttmjBhgu677z596Utf0rPPPquHH35YDz/8sCSpoKBAs2bN0je/+U0NHz5cQ4YM0Te+8Q2Vl5dr6tSpHdF/AAAAHCNYohOPdk3wzznnHK1YsUJz5szRPffcoyFDhmjhwoW65pprsjV33HGHGhsbddNNN6murk4TJ07UqlWr1L27mxD0kVeUM2bDCUaRpEP/bW9bXYyICmflkJvl5QTb1Bk17rOdnJAcp71is72oQl2cfewmsTjHgRMA5ITouNty9p97nDv7uNCoiSoISvJSZJwxcNtz/j7fCcNy9p0bUuZc9pwxd4/znkbNy0ZNH7O90vwlVhCUc6y4USrO76Wdc8EdA4dzHNSZ2zL63sM5XpwxN68/3Yzgqfed7XjNWZmLUeUW1hk1rvbOgnJ5LcdrzZL+GGFbODa1O8n27/7u7/R3f/d3h329oKBA99xzj+65556j6hgAAACA9mv3BB8AAAA4EhklK1wq09kd6CDt+iNbAAAAAMnGBB8AAAAICEt0AAAAEAueohMP7uADAAAAAWGCDwAAAASEJToAAACIBUt04pHcCf72/5R653jdCaORpD1GTb1RY2Rh6SmjRoou1MXZjuQdvc54ukEzzvOvnKCgIqPG2XeSl1bipJBsM9trNmqcMBo3+cXZf064j3O+OGFRUnRBOu7VN6r9l+u68xEn4EnyAt3c89jxulHjBPK511cnxKrcqIlyDJw+OYFgb5rtGSFP1vXOTUFyxsp5f86109l3kgqMMehlXBNL3jHbM0IXhxrvz/moMofACvJ6zahx8zJzcT+qETaW6AAAAAABSe4dfAAAAASlVckKukpSX6LEHXwAAAAgIEzwAQAAgICwRAcAAACx4Ck68eAOPgAAABAQJvgAAABAQFiiAwAAgFjwFJ14cAcfAAAACEhy7+DvVu6Uy43mdpxEPyeCzkkidFNHXzFqokqDlSQj9c9KenWSVyVvzJ2kTCdR1B0Dh9OntLmtN4yaBqPGTbeMirOP3b9IctJ8o0w53WrUVBg1rxk1JUaN5F0TnGPYjdOsNmpSRo1z7kne8emk+ToJymVGjeTvm3zc49xJsnW25aQ6S97nldOeM57OZ5XkXReNc6Gfe70zxmCgc+4Zx3mzc/zKG/K0UVPjNadBOV5rlPQtczsIV3In+AAAAAgKT9GJB0t0AAAAgIAwwQcAAAACwhIdAAAAxKJVyVoWw1N0AAAAACQeE3wAAAAgICzRAQAAQCwIuopH4ib4mUxGktSQ75nA+80NFhg1znOFnfac7bjbco445/nLbnvOtqIcc6c95znG7hhEtS13DJxtOc+Jj/t3bM5z8N2rofP+nHFyn4Pv1EXVnnvcOT4wapyxlLy+R1UjeceL8/6cBblun6LiLhKO6v257UW1Laff7rnu1EVVI0mZ+GrcU88ZcifyYp/ZXq5L0EfTp4/mUzg+JW6Cv3fvh6dAxeWd3BEASIr/29kdAHIww6AS573O7kDH2rt3r4qLnaRLhChxE/zy8nJVV1erd+/eKij48FZwQ0ODKioqVF1drVTKiWHE0WLM48eYx4vxjh9jHj/GPH6dPeaZTEZ79+5VebkbgR2vFiXrD0CT9ESfKCVugt+lSxcNGDDgkK+lUikuUDFjzOPHmMeL8Y4fYx4/xjx+nTnm3LlHkn6IAgAAAHCUEncHHwAAAGFiiU48kjTGh1VUVKR58+apqKios7ty3GDM48eYx4vxjh9jHj/GPH6MOZKgIMNzlAAAANCBGhoaVFxcrL+T9+TquByQ9GtJ9fX1Qf2dCkt0AAAAEAuCruJxTCzRAQAAAOBhgg8AAAAEhCU6AAAAiAVP0YlHksb4kBYtWqTBgwere/fuGj9+vJ599tnO7lJQnn76aV166aUqLy9XQUGBVq5c2eb1TCajuXPnqn///urRo4eqqqr0yiuvdE5nAzB//nydc8456t27t/r166epU6dq69atbWr279+vGTNmqE+fPjrxxBM1bdo01dbWdlKPj30PPfSQRo8enQ2dqays1BNPPJF9nfHuWN/+9rdVUFCgWbNmZb/HmEfrrrvuUkFBQZuvkSNHZl9nvDvGm2++qf/xP/6H+vTpox49euiMM87Qpk2bsq/z+YnOlOgJ/s9//nPNnj1b8+bN0/PPP68xY8Zo8uTJ2r17d2d3LRiNjY0aM2aMFi1adMjXv/vd7+qBBx7Q4sWLtXHjRvXq1UuTJ0/W/v37Y+5pGNatW6cZM2Zow4YNWr16tQ4cOKCLLrpIjY2N2ZrbbrtNjz/+uJYvX65169Zp165duvzyyzux18e2AQMG6Nvf/rY2b96sTZs26YILLtBll12m//zP/5TEeHek5557Tj/84Q81evToNt9nzKP36U9/Wm+99Vb26w9/+EP2NcY7eu+9957OP/98devWTU888YReeukl3X///TrppJOyNXx+olNlEuzcc8/NzJgxI/v/LS0tmfLy8sz8+fM7sVfhkpRZsWJF9v9bW1szZWVlme9973vZ79XV1WWKiooy//qv/9oJPQzP7t27M5Iy69aty2QyH45vt27dMsuXL8/W/PnPf85Iyqxfv76zuhmck046KfPjH/+Y8e5Ae/fuzQwfPjyzevXqzOc+97nMrbfemslkOMY7wrx58zJjxow55GuMd8f42te+lpk4ceJhX+fz82D19fUZSZkvSJkvJujrC1JGUqa+vr6zhyhSib2D39zcrM2bN6uqqir7vS5duqiqqkrr16/vxJ4dP3bs2KGampo2+6C4uFjjx49nH0Skvr5eklRSUiJJ2rx5sw4cONBmzEeOHKmBAwcy5hFoaWnRsmXL1NjYqMrKSsa7A82YMUOXXHJJm7GVOMY7yiuvvKLy8nINHTpU11xzjXbu3CmJ8e4ov/rVrzRu3DhdccUV6tevn8466yz96Ec/yr7O5yck6Vvf+pYmTJignj17Kp1OW//muuuuO2jJ3cUXX9zuthM7wX/nnXfU0tKi0tLSNt8vLS1VTU1NJ/Xq+PLROLMPOkZra6tmzZql888/X6effrqkD8e8sLDwoAsBY350XnjhBZ144okqKirSl7/8Za1YsUKnnXYa491Bli1bpueff17z588/6DXGPHrjx4/XkiVLtGrVKj300EPasWOHPvOZz2jv3r2MdwfZvn27HnroIQ0fPlxPPvmkbr75Zn3lK1/RT3/6U0l8fuJDzc3NuuKKK3TzzTe3699dfPHFbZbc/eu//mu72+YpOkAnmTFjhl588cU2a2XRMUaMGKEtW7aovr5e//Zv/6bp06dr3bp1nd2tIFVXV+vWW2/V6tWr1b17987uznFhypQp2f8ePXq0xo8fr0GDBukXv/iFevTo0Yk9C1dra6vGjRun++67T5J01lln6cUXX9TixYs1ffr0Tu5dsrUqWU+u6cigq7vvvluStGTJknb9u6KiIpWVlR1V24m9g3/yySera9euB/2lf21t7VG/aXg+Gmf2QfRmzpypX//61/r973+vAQMGZL9fVlam5uZm1dXVtalnzI9OYWGhhg0bprFjx2r+/PkaM2aMfvCDHzDeHWDz5s3avXu3zj77bJ1wwgk64YQTtG7dOj3wwAM64YQTVFpayph3sHQ6rU996lPatm0bx3gH6d+/v0477bQ23xs1alR2aRSfnzgaa9euVb9+/TRixAjdfPPN2rNnT7u3kdgJfmFhocaOHas1a9Zkv9fa2qo1a9aosrKyE3t2/BgyZIjKysra7IOGhgZt3LiRfXCEMpmMZs6cqRUrVuipp57SkCFD2rw+duxYdevWrc2Yb926VTt37mTMI9Ta2qqmpibGuwNceOGFeuGFF7Rly5bs17hx43TNNddk/5sx71j79u3Tq6++qv79+3OMd5Dzzz//oEcc/+Uvf9GgQYMk8fl5LGpoaGjz1dTU1Cn9uPjii/XYY49pzZo1+s53vqN169ZpypQpamlp3+89Er1EZ/bs2Zo+fbrGjRunc889VwsXLlRjY6Ouv/76zu5aMPbt26dt27Zl/3/Hjh3asmWLSkpKNHDgQM2aNUvf/OY3NXz4cA0ZMkTf+MY3VF5erqlTp3Zep49hM2bM0NKlS/XLX/5SvXv3zq7FLC4uVo8ePVRcXKwbbrhBs2fPVklJiVKplG655RZVVlbqvPPO6+TeH5vmzJmjKVOmaODAgdq7d6+WLl2qtWvX6sknn2S8O0Dv3r2zf1PykV69eqlPnz7Z7zPm0frqV7+qSy+9VIMGDdKuXbs0b948de3aVVdffTXHeAe57bbbNGHCBN1333360pe+pGeffVYPP/ywHn74YUnKZj/w+XmwFkkFnd2JT/ho2lxRUdHm+/PmzdNdd911UP3Xv/51fec738m5zT//+c9tsija46qrrsr+9xlnnKHRo0fr1FNP1dq1a3XhhRfa20n0BP/KK6/U22+/rblz56qmpkZnnnmmVq1addAfreDIbdq0SZ///Oez/z979mxJ0vTp07VkyRLdcccdamxs1E033aS6ujpNnDhRq1atYm3tEXrooYckSZMmTWrz/UcffVTXXXedJGnBggXq0qWLpk2bpqamJk2ePFkPPvhgzD0Nx+7du3XttdfqrbfeUnFxsUaPHq0nn3xSX/jCFyQx3p2BMY/WG2+8oauvvlp79uxR3759NXHiRG3YsEF9+/aVxHh3hHPOOUcrVqzQnDlzdM8992jIkCFauHChrrnmmmwNn5/HlurqaqVSqez/FxUVHbLu9ttvz35eH87QoUMj69fQoUN18skna9u2be2a4BdkMplMZL0AAAAA/kZDQ4OKi4s1Scm6u/yBpLX68LHVn5zgR2nJkiWaNWvWQX8L43jjjTc0cOBArVy5Uv/tv/03+98ldg0+AAAAwtKawK+OsnPnTm3ZskU7d+5US0tL9u+S9u3bl60ZOXKkVqxYIenDZdP/8A//oA0bNui1117TmjVrdNlll2nYsGGaPHlyu9pO0g9RAAAAQBDmzp2bzUaQPnycqiT9/ve/zy7V3bp1azb0smvXrvrTn/6kn/70p6qrq1N5ebkuuugi3XvvvYddMnQ4LNEBAABAh/poic5nlay7yx9Ielodu0SnMyRpjAEAABCwpD5FJzSswQcAAAACwgQfAAAACAhLdAAAABALlujEgzv4AAAAQECY4AMAAAABYYkOAAAAYtHR4VLtlaS+RIk7+AAAAEBAmOADAAAAAWGJDgAAAGLBU3TiwR18AAAAICBM8AEAAICAsEQHAAAAscgoWU+uyXR2BzoId/ABAACAgDDBBwAAAALCEh0AAADEImlPrUlaf6LCHXwAAAAgIEzwAQAAgICwRAcAAACxSNqSmKT1JyrcwQcAAAACwgQfAAAACAhLdAAAABCLVkkFnd2JT0hS6FaUuIMPAAAABIQJPgAAABAQlugAAAAgFkl7ak3S+hMV7uADAAAAAWGCDwAAAASEJToAAACIRdKWxCStP1HhDj4AAAAQECb4AAAAQEBYogMAAIBYEHQVD+7gAwAAAAFhgg8AAAAEhCU6AAAAiEXSlsQkrT9R4Q4+AAAAEBAm+AAAAEBAWKIDAACAWCRtSUzS+hMV7uADAAAAAWGCDwAAAASEJToAAACIRYukTGd34hNYogMAAAAg8ZjgAwAAAAFhiQ4AAABiwRKdeHAHHwAAAAgIE3wAAAAgICzRAQAAQCyStiQmaf2JCnfwAQAAgIAwwQcAAAACwhIdAAAAxIKn6MSDO/gAAABAQJjgAwAAAAFhiQ4AAABi0apkLdFJUl+ixB18AAAAICBM8AEAAICAsEQHAAAAsWiVVNDZnfgElugAAAAASDwm+AAAAEBAWKIDAACAWLSIJTpx4A4+AAAAEBAm+AAAAEBAWKIDAACAWPAUnXhwBx8AAAAICHfwAQAAEIuk3TFPWn+iwgQfAAAAHaqwsFBlZWWqqanp7K4cpKysTIWFhZ3djUgVZDKZUH94AQAAQELs379fzc3Nnd2NgxQWFqp79+6d3Y1IMcEHAAAAAsIf2QIAAAABYYIPAAAABIQJPgAAABAQJvgAAABAQJjgAwAAAAFhgg8AAAAEhAk+AAAAEJD/H8KXGmvg1TAgAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 800x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "attn = model.blocks[0].attn\n",
    "all_token_embeddings = model.embed(range(D_VOCAB))\n",
    "\n",
    "embeddings_key = einsum(\"d_vocab d_model, n_heads d_model d_head -> n_heads d_vocab d_head\", \n",
    "                        all_token_embeddings, attn.W_K) \n",
    "embeddings_query = einsum(\"d_vocab d_model, n_heads d_model d_head -> n_heads d_vocab d_head\", \n",
    "                          all_token_embeddings, attn.W_Q) \n",
    "                          \n",
    "plt.rcParams['figure.figsize'] = [20, 10]\n",
    "qk_circuit_attn_heatmap = einsum(\n",
    "    \"n_heads d_vocab_q d_head, n_heads d_vocab_k d_head -> ... d_vocab_q d_vocab_k\", \n",
    "    embeddings_query, embeddings_key\n",
    "    ).detach().cpu().numpy()\n",
    "fig, ax = plt.subplots(figsize=(8, 8))\n",
    "graph = ax.imshow(qk_circuit_attn_heatmap, cmap=\"hot\", interpolation=\"nearest\")\n",
    "plt.colorbar(graph)\n",
    "plt.tight_layout()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}