knn_classification.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_d5EUe2h4jGu"
      },
      "source": [
        "\n",
        "What Can Transformers Learn In-Context? A Case Study of Simple Function Classes\n",
        "\n",
        "Shivam Garg*, Dimitris Tsipras*, Percy Liang, Gregory Valiant\n",
        "\n",
        "Paper: http://arxiv.org/abs/2208.01066\n",
        "\n",
        "\n",
        "https://github.com/dtsip/in-context-learning.git\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XgwnF6TW40UT"
      },
      "source": [
        "#Prerequisites\n",
        "\n",
        "tested on Python 3.10 with a T4 GPU"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "prnYu7oW6Usg"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install -q matplotlib==3.5.2 numpy==1.22.3 pandas==1.4.2 scikit-learn==1.0.2 seaborn==0.11.2 tqdm==4.64.0 transformers==4.17.0 wandb==0.12.11 xgboost==1.6.1 protobuf==3.20.1 torch==2.1.0+cu118"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jOAqiGNaqBrS"
      },
      "source": [
        "#Config\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "T41GRct3BfCa"
      },
      "outputs": [],
      "source": [
        "class args_c():\n",
        "    class wandb_c():\n",
        "        def __init__(self):             # wandb\n",
        "            self.project = \"in-context-learning\"\n",
        "            self.name = \"linear_regression_toy\"\n",
        "            self.entity = \"your-entity\"\n",
        "            self.notes = \"\"\n",
        "            self.log_every_steps = 100\n",
        "\n",
        "    class model_c():\n",
        "        def __init__(self):             # model\n",
        "            self.family = \"gpt2\"\n",
        "            self.n_embd = 256\n",
        "            self.n_layer = 12\n",
        "            self.n_head = 8\n",
        "            # toy: n_dims = 5, n_positions = 11\n",
        "            # full: n_dims = 20, n_positions = 101\n",
        "            self.n_dims = 11\n",
        "            self.n_positions = 49\n",
        "\n",
        "    class training_c():\n",
        "        class curriculum_c():\n",
        "            class dims_c():\n",
        "                def __init__(self):     # dims\n",
        "                    self.start = 5\n",
        "                    self.end = 11\n",
        "                    self.inc = 1\n",
        "                    self.interval = 500\n",
        "\n",
        "            class points_c():\n",
        "                def __init__(self):     # points\n",
        "                    self.start = 11\n",
        "                    self.end = 41\n",
        "                    self.inc = 2\n",
        "                    self.interval = 100\n",
        "\n",
        "            def __init__(self):         # curriculum\n",
        "                self.dims = self.dims_c()\n",
        "                self.points = self.points_c()\n",
        "\n",
        "        def __init__(self):             # training\n",
        "            self.task = \"knn_classification\"\n",
        "            self.data = \"gaussian\"\n",
        "            self.task_kwargs = {\"k\":1, \"num_pregeneration\":10, \"num_labels\":2}\n",
        "            self.num_tasks = None\n",
        "            self.num_training_examples = None\n",
        "            self.batch_size = 32\n",
        "            self.learning_rate = .0001\n",
        "            self.save_every_steps = 1000\n",
        "            self.keep_every_steps = 100000\n",
        "            self.train_steps = 5001\n",
        "            self.resume_id = None\n",
        "\n",
        "            self.curriculum = self.curriculum_c()\n",
        "\n",
        "    def __init__(self):                 # args\n",
        "        self.wandb = self.wandb_c()\n",
        "        self.model = self.model_c()\n",
        "        self.training = self.training_c()\n",
        "\n",
        "        self.out_dir = \"./models/test\"\n",
        "        self.test_run = False\n",
        "        self.using_wandb = False\n",
        "\n",
        "args = args_c()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x4xotR5ai-0m"
      },
      "source": [
        "#Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "8qZ_eCxA5hin"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import GPT2Model, GPT2Config\n",
        "from tqdm import tqdm\n",
        "from sklearn.svm import LinearSVC\n",
        "from sklearn.linear_model import LogisticRegression, Lasso\n",
        "import warnings\n",
        "from sklearn import tree\n",
        "import xgboost as xgb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "25rFZAws0Fsc"
      },
      "outputs": [],
      "source": [
        "def build_model(conf):\n",
        "    if conf.family == \"gpt2\":\n",
        "        model = TransformerModel(\n",
        "            n_dims=conf.n_dims,\n",
        "            n_positions=conf.n_positions,\n",
        "            n_embd=conf.n_embd,\n",
        "            n_layer=conf.n_layer,\n",
        "            n_head=conf.n_head,\n",
        "        )\n",
        "    else:\n",
        "        raise NotImplementedError\n",
        "\n",
        "    return model\n",
        "\n",
        "\n",
        "def get_relevant_baselines(task_name):\n",
        "    task_to_baselines = {\n",
        "        \"knn_classification\": [\n",
        "            (LeastSquaresModel, {}),\n",
        "            (NNModel, {\"n_neighbors\": 3}),\n",
        "            # (DecisionTreeModel, {\"max_depth\": 4}),\n",
        "            # (DecisionTreeModel, {\"max_depth\": None}),\n",
        "            # (XGBoostModel, {}),\n",
        "            (AveragingModel, {}),\n",
        "        ],\n",
        "    }\n",
        "\n",
        "    models = [model_cls(**kwargs) for model_cls, kwargs in task_to_baselines[task_name]]\n",
        "    return models\n",
        "\n",
        "\n",
        "class NeuralNetwork(nn.Module):\n",
        "    def __init__(self, in_size=50, hidden_size=1000, out_size=1):\n",
        "        super(NeuralNetwork, self).__init__()\n",
        "\n",
        "        self.net = nn.Sequential(\n",
        "            nn.Linear(in_size, hidden_size),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(hidden_size, out_size),\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = self.net(x)\n",
        "        return out\n",
        "\n",
        "\n",
        "class ParallelNetworks(nn.Module):\n",
        "    def __init__(self, num_models, model_class, **model_class_init_args):\n",
        "        super(ParallelNetworks, self).__init__()\n",
        "        self.nets = nn.ModuleList(\n",
        "            [model_class(**model_class_init_args) for i in range(num_models)]\n",
        "        )\n",
        "\n",
        "    def forward(self, xs):\n",
        "        assert xs.shape[0] == len(self.nets)\n",
        "\n",
        "        for i in range(len(self.nets)):\n",
        "            out = self.nets[i](xs[i])\n",
        "            if i == 0:\n",
        "                outs = torch.zeros(\n",
        "                    [len(self.nets)] + list(out.shape), device=out.device\n",
        "                )\n",
        "            outs[i] = out\n",
        "        return outs\n",
        "\n",
        "class TransformerModel(nn.Module):\n",
        "    def __init__(self, n_dims, n_positions, n_embd=128, n_layer=12, n_head=4):\n",
        "        super(TransformerModel, self).__init__()\n",
        "        configuration = GPT2Config(\n",
        "            n_positions=2 * n_positions,\n",
        "            n_embd=n_embd,\n",
        "            n_layer=n_layer,\n",
        "            n_head=n_head,\n",
        "            resid_pdrop=0.0,\n",
        "            embd_pdrop=0.0,\n",
        "            attn_pdrop=0.0,\n",
        "            use_cache=False,\n",
        "        )\n",
        "        self.name = f\"gpt2_embd={n_embd}_layer={n_layer}_head={n_head}\"\n",
        "\n",
        "        self.n_positions = n_positions\n",
        "        self.n_dims = n_dims\n",
        "        self._read_in = nn.Linear(n_dims, n_embd)\n",
        "        self._backbone = GPT2Model(configuration)\n",
        "        self._read_out = nn.Linear(n_embd, 1)\n",
        "\n",
        "    @staticmethod\n",
        "    def _combine(xs_b, ys_b):\n",
        "        \"\"\"Interleaves the x's and the y's into a single sequence.\"\"\"\n",
        "        bsize, points, dim = xs_b.shape\n",
        "        ys_b_wide = torch.cat(\n",
        "            (\n",
        "                ys_b.view(bsize, points, 1),\n",
        "                torch.zeros(bsize, points, dim - 1, device=ys_b.device),\n",
        "            ),\n",
        "            axis=2,\n",
        "        )\n",
        "        zs = torch.stack((xs_b, ys_b_wide), dim=2)\n",
        "        zs = zs.view(bsize, 2 * points, dim)\n",
        "        return zs\n",
        "\n",
        "    def forward(self, xs, ys, inds=None):\n",
        "        if inds is None:\n",
        "            inds = torch.arange(ys.shape[1])\n",
        "        else:\n",
        "            inds = torch.tensor(inds)\n",
        "            if max(inds) >= ys.shape[1] or min(inds) < 0:\n",
        "                raise ValueError(\"inds contain indices where xs and ys are not defined\")\n",
        "        zs = self._combine(xs, ys)\n",
        "        embeds = self._read_in(zs)\n",
        "        output = self._backbone(inputs_embeds=embeds).last_hidden_state\n",
        "        prediction = self._read_out(output)\n",
        "        return prediction[:, ::2, 0][:, inds]  # predict only on xs\n",
        "\n",
        "\n",
        "class NNModel:\n",
        "    def __init__(self, n_neighbors, weights=\"uniform\"):\n",
        "        # should we be picking k optimally\n",
        "        self.n_neighbors = n_neighbors\n",
        "        self.weights = weights\n",
        "        self.name = f\"NN_n={n_neighbors}_{weights}\"\n",
        "\n",
        "    def __call__(self, xs, ys, inds=None):\n",
        "        if inds is None:\n",
        "            inds = range(ys.shape[1])\n",
        "        else:\n",
        "            if max(inds) >= ys.shape[1] or min(inds) < 0:\n",
        "                raise ValueError(\"inds contain indices where xs and ys are not defined\")\n",
        "\n",
        "        preds = []\n",
        "\n",
        "        for i in inds:\n",
        "            if i == 0:\n",
        "                preds.append(torch.zeros_like(ys[:, 0]))  # predict zero for first point\n",
        "                continue\n",
        "            train_xs, train_ys = xs[:, :i], ys[:, :i]\n",
        "            test_x = xs[:, i : i + 1]\n",
        "            dist = (train_xs - test_x).square().sum(dim=2).sqrt()\n",
        "\n",
        "            if self.weights == \"uniform\":\n",
        "                weights = torch.ones_like(dist)\n",
        "            else:\n",
        "                weights = 1.0 / dist\n",
        "                inf_mask = torch.isinf(weights).float()  # deal with exact match\n",
        "                inf_row = torch.any(inf_mask, axis=1)\n",
        "                weights[inf_row] = inf_mask[inf_row]\n",
        "\n",
        "            pred = []\n",
        "            k = min(i, self.n_neighbors)\n",
        "            ranks = dist.argsort()[:, :k]\n",
        "            for y, w, n in zip(train_ys, weights, ranks):\n",
        "                y, w = y[n], w[n]\n",
        "                pred.append((w * y).sum() / w.sum())\n",
        "            preds.append(torch.stack(pred))\n",
        "\n",
        "        return torch.stack(preds, dim=1)\n",
        "\n",
        "\n",
        "# xs and ys should be on cpu for this method. Otherwise the output maybe off in case when train_xs is not full rank due to the implementation of torch.linalg.lstsq.\n",
        "class LeastSquaresModel:\n",
        "    def __init__(self, driver=None):\n",
        "        self.driver = driver\n",
        "        self.name = f\"OLS_driver={driver}\"\n",
        "\n",
        "    def __call__(self, xs, ys, inds=None):\n",
        "        xs, ys = xs.cpu(), ys.cpu()\n",
        "        if inds is None:\n",
        "            inds = range(ys.shape[1])\n",
        "        else:\n",
        "            if max(inds) >= ys.shape[1] or min(inds) < 0:\n",
        "                raise ValueError(\"inds contain indices where xs and ys are not defined\")\n",
        "\n",
        "        preds = []\n",
        "\n",
        "        for i in inds:\n",
        "            if i == 0:\n",
        "                preds.append(torch.zeros_like(ys[:, 0]))  # predict zero for first point\n",
        "                continue\n",
        "            train_xs, train_ys = xs[:, :i], ys[:, :i]\n",
        "            test_x = xs[:, i : i + 1]\n",
        "\n",
        "            ws, _, _, _ = torch.linalg.lstsq(\n",
        "                train_xs, train_ys.unsqueeze(2), driver=self.driver\n",
        "            )\n",
        "\n",
        "            pred = test_x @ ws\n",
        "            preds.append(pred[:, 0, 0])\n",
        "\n",
        "        return torch.stack(preds, dim=1)\n",
        "\n",
        "\n",
        "class AveragingModel:\n",
        "    def __init__(self):\n",
        "        self.name = \"averaging\"\n",
        "\n",
        "    def __call__(self, xs, ys, inds=None):\n",
        "        if inds is None:\n",
        "            inds = range(ys.shape[1])\n",
        "        else:\n",
        "            if max(inds) >= ys.shape[1] or min(inds) < 0:\n",
        "                raise ValueError(\"inds contain indices where xs and ys are not defined\")\n",
        "\n",
        "        preds = []\n",
        "\n",
        "        for i in inds:\n",
        "            if i == 0:\n",
        "                preds.append(torch.zeros_like(ys[:, 0]))  # predict zero for first point\n",
        "                continue\n",
        "            train_xs, train_ys = xs[:, :i], ys[:, :i]\n",
        "            test_x = xs[:, i : i + 1]\n",
        "\n",
        "            train_zs = train_xs * train_ys.unsqueeze(dim=-1)\n",
        "            w_p = train_zs.mean(dim=1).unsqueeze(dim=-1)\n",
        "            pred = test_x @ w_p\n",
        "            preds.append(pred[:, 0, 0])\n",
        "\n",
        "        return torch.stack(preds, dim=1)\n",
        "\n",
        "\n",
        "# Lasso regression (for sparse linear regression).\n",
        "# Seems to take more time as we decrease alpha.\n",
        "class LassoModel:\n",
        "    def __init__(self, alpha, max_iter=100000):\n",
        "        # the l1 regularizer gets multiplied by alpha.\n",
        "        self.alpha = alpha\n",
        "        self.max_iter = max_iter\n",
        "        self.name = f\"lasso_alpha={alpha}_max_iter={max_iter}\"\n",
        "\n",
        "    # inds is a list containing indices where we want the prediction.\n",
        "    # prediction made at all indices by default.\n",
        "    def __call__(self, xs, ys, inds=None):\n",
        "        xs, ys = xs.cpu(), ys.cpu()\n",
        "\n",
        "        if inds is None:\n",
        "            inds = range(ys.shape[1])\n",
        "        else:\n",
        "            if max(inds) >= ys.shape[1] or min(inds) < 0:\n",
        "                raise ValueError(\"inds contain indices where xs and ys are not defined\")\n",
        "\n",
        "        preds = []  # predict one for first point\n",
        "\n",
        "        # i: loop over num_points\n",
        "        # j: loop over bsize\n",
        "        for i in inds:\n",
        "            pred = torch.zeros_like(ys[:, 0])\n",
        "\n",
        "            if i > 0:\n",
        "                pred = torch.zeros_like(ys[:, 0])\n",
        "                for j in range(ys.shape[0]):\n",
        "                    train_xs, train_ys = xs[j, :i], ys[j, :i]\n",
        "\n",
        "                    # If all points till now have the same label, predict that label.\n",
        "\n",
        "                    clf = Lasso(\n",
        "                        alpha=self.alpha, fit_intercept=False, max_iter=self.max_iter\n",
        "                    )\n",
        "\n",
        "                    # Check for convergence.\n",
        "                    with warnings.catch_warnings():\n",
        "                        warnings.filterwarnings(\"error\")\n",
        "                        try:\n",
        "                            clf.fit(train_xs, train_ys)\n",
        "                        except Warning:\n",
        "                            print(f\"lasso convergence warning at i={i}, j={j}.\")\n",
        "                            raise\n",
        "\n",
        "                    w_pred = torch.from_numpy(clf.coef_).unsqueeze(1)\n",
        "\n",
        "                    test_x = xs[j, i : i + 1]\n",
        "                    y_pred = (test_x @ w_pred.float()).squeeze(1)\n",
        "                    pred[j] = y_pred[0]\n",
        "\n",
        "            preds.append(pred)\n",
        "\n",
        "        return torch.stack(preds, dim=1)\n",
        "\n",
        "\n",
        "# Gradient Descent and variants.\n",
        "# Example usage: gd_model = GDModel(NeuralNetwork, {'in_size': 50, 'hidden_size':400, 'out_size' :1}, opt_alg = 'adam', batch_size = 100, lr = 5e-3, num_steps = 200)\n",
        "class GDModel:\n",
        "    def __init__(\n",
        "        self,\n",
        "        model_class,\n",
        "        model_class_args,\n",
        "        opt_alg=\"sgd\",\n",
        "        batch_size=1,\n",
        "        num_steps=1000,\n",
        "        lr=1e-3,\n",
        "        loss_name=\"squared\",\n",
        "    ):\n",
        "        # model_class: torch.nn model class\n",
        "        # model_class_args: a dict containing arguments for model_class\n",
        "        # opt_alg can be 'sgd' or 'adam'\n",
        "        # verbose: whether to print the progress or not\n",
        "        # batch_size: batch size for sgd\n",
        "        self.model_class = model_class\n",
        "        self.model_class_args = model_class_args\n",
        "        self.opt_alg = opt_alg\n",
        "        self.lr = lr\n",
        "        self.batch_size = batch_size\n",
        "        self.num_steps = num_steps\n",
        "        self.loss_name = loss_name\n",
        "\n",
        "        self.name = f\"gd_model_class={model_class}_model_class_args={model_class_args}_opt_alg={opt_alg}_lr={lr}_batch_size={batch_size}_num_steps={num_steps}_loss_name={loss_name}\"\n",
        "\n",
        "    def __call__(self, xs, ys, inds=None, verbose=False, print_step=100):\n",
        "        # inds is a list containing indices where we want the prediction.\n",
        "        # prediction made at all indices by default.\n",
        "        # xs: bsize X npoints X ndim.\n",
        "        # ys: bsize X npoints.\n",
        "        xs, ys = xs.cuda(), ys.cuda()\n",
        "\n",
        "        if inds is None:\n",
        "            inds = range(ys.shape[1])\n",
        "        else:\n",
        "            if max(inds) >= ys.shape[1] or min(inds) < 0:\n",
        "                raise ValueError(\"inds contain indices where xs and ys are not defined\")\n",
        "\n",
        "        preds = []  # predict one for first point\n",
        "\n",
        "        # i: loop over num_points\n",
        "        for i in tqdm(inds):\n",
        "            pred = torch.zeros_like(ys[:, 0])\n",
        "            model = ParallelNetworks(\n",
        "                ys.shape[0], self.model_class, **self.model_class_args\n",
        "            )\n",
        "            model.cuda()\n",
        "            if i > 0:\n",
        "                pred = torch.zeros_like(ys[:, 0])\n",
        "\n",
        "                train_xs, train_ys = xs[:, :i], ys[:, :i]\n",
        "                test_xs, test_ys = xs[:, i : i + 1], ys[:, i : i + 1]\n",
        "\n",
        "                if self.opt_alg == \"sgd\":\n",
        "                    optimizer = torch.optim.SGD(model.parameters(), lr=self.lr)\n",
        "                elif self.opt_alg == \"adam\":\n",
        "                    optimizer = torch.optim.Adam(model.parameters(), lr=self.lr)\n",
        "                else:\n",
        "                    raise NotImplementedError(f\"{self.opt_alg} not implemented.\")\n",
        "\n",
        "                if self.loss_name == \"squared\":\n",
        "                    loss_criterion = nn.MSELoss()\n",
        "                else:\n",
        "                    raise NotImplementedError(f\"{self.loss_name} not implemented.\")\n",
        "\n",
        "                # Training loop\n",
        "                for j in range(self.num_steps):\n",
        "\n",
        "                    # Prepare batch\n",
        "                    mask = torch.zeros(i).bool()\n",
        "                    perm = torch.randperm(i)\n",
        "                    mask[perm[: self.batch_size]] = True\n",
        "                    train_xs_cur, train_ys_cur = train_xs[:, mask, :], train_ys[:, mask]\n",
        "\n",
        "                    if verbose and j % print_step == 0:\n",
        "                        model.eval()\n",
        "                        with torch.no_grad():\n",
        "                            outputs = model(train_xs_cur)\n",
        "                            loss = loss_criterion(\n",
        "                                outputs[:, :, 0], train_ys_cur\n",
        "                            ).detach()\n",
        "                            outputs_test = model(test_xs)\n",
        "                            test_loss = loss_criterion(\n",
        "                                outputs_test[:, :, 0], test_ys\n",
        "                            ).detach()\n",
        "                            print(\n",
        "                                f\"ind:{i},step:{j}, train_loss:{loss.item()}, test_loss:{test_loss.item()}\"\n",
        "                            )\n",
        "\n",
        "                    optimizer.zero_grad()\n",
        "\n",
        "                    model.train()\n",
        "                    outputs = model(train_xs_cur)\n",
        "                    loss = loss_criterion(outputs[:, :, 0], train_ys_cur)\n",
        "                    loss.backward()\n",
        "                    optimizer.step()\n",
        "\n",
        "                model.eval()\n",
        "                pred = model(test_xs).detach()\n",
        "\n",
        "                assert pred.shape[1] == 1 and pred.shape[2] == 1\n",
        "                pred = pred[:, 0, 0]\n",
        "\n",
        "            preds.append(pred)\n",
        "\n",
        "        return torch.stack(preds, dim=1)\n",
        "\n",
        "\n",
        "class DecisionTreeModel:\n",
        "    def __init__(self, max_depth=None):\n",
        "        self.max_depth = max_depth\n",
        "        self.name = f\"decision_tree_max_depth={max_depth}\"\n",
        "\n",
        "    # inds is a list containing indices where we want the prediction.\n",
        "    # prediction made at all indices by default.\n",
        "    def __call__(self, xs, ys, inds=None):\n",
        "        xs, ys = xs.cpu(), ys.cpu()\n",
        "\n",
        "        if inds is None:\n",
        "            inds = range(ys.shape[1])\n",
        "        else:\n",
        "            if max(inds) >= ys.shape[1] or min(inds) < 0:\n",
        "                raise ValueError(\"inds contain indices where xs and ys are not defined\")\n",
        "\n",
        "        preds = []\n",
        "\n",
        "        # i: loop over num_points\n",
        "        # j: loop over bsize\n",
        "        for i in inds:\n",
        "            pred = torch.zeros_like(ys[:, 0])\n",
        "\n",
        "            if i > 0:\n",
        "                pred = torch.zeros_like(ys[:, 0])\n",
        "                for j in range(ys.shape[0]):\n",
        "                    train_xs, train_ys = xs[j, :i], ys[j, :i]\n",
        "\n",
        "                    clf = tree.DecisionTreeRegressor(max_depth=self.max_depth)\n",
        "                    clf = clf.fit(train_xs, train_ys)\n",
        "                    test_x = xs[j, i : i + 1]\n",
        "                    y_pred = clf.predict(test_x)\n",
        "                    pred[j] = y_pred[0]\n",
        "\n",
        "            preds.append(pred)\n",
        "\n",
        "        return torch.stack(preds, dim=1)\n",
        "\n",
        "\n",
        "class XGBoostModel:\n",
        "    def __init__(self):\n",
        "        self.name = \"xgboost\"\n",
        "\n",
        "    # inds is a list containing indices where we want the prediction.\n",
        "    # prediction made at all indices by default.\n",
        "    def __call__(self, xs, ys, inds=None):\n",
        "        xs, ys = xs.cpu(), ys.cpu()\n",
        "\n",
        "        if inds is None:\n",
        "            inds = range(ys.shape[1])\n",
        "        else:\n",
        "            if max(inds) >= ys.shape[1] or min(inds) < 0:\n",
        "                raise ValueError(\"inds contain indices where xs and ys are not defined\")\n",
        "\n",
        "        preds = []\n",
        "\n",
        "        # i: loop over num_points\n",
        "        # j: loop over bsize\n",
        "        for i in tqdm(inds):\n",
        "            pred = torch.zeros_like(ys[:, 0])\n",
        "            if i > 0:\n",
        "                pred = torch.zeros_like(ys[:, 0])\n",
        "                for j in range(ys.shape[0]):\n",
        "                    train_xs, train_ys = xs[j, :i], ys[j, :i]\n",
        "\n",
        "                    clf = xgb.XGBRegressor()\n",
        "\n",
        "                    clf = clf.fit(train_xs, train_ys)\n",
        "                    test_x = xs[j, i : i + 1]\n",
        "                    y_pred = clf.predict(test_x)\n",
        "                    pred[j] = y_pred[0].item()\n",
        "\n",
        "            preds.append(pred)\n",
        "\n",
        "        return torch.stack(preds, dim=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JcP-QF8BDpDC"
      },
      "source": [
        "#Samplers\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "0QzD2i99D8_y"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "\n",
        "import torch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "aNuGvNB1Dqdu"
      },
      "outputs": [],
      "source": [
        "class DataSampler:\n",
        "    def __init__(self, n_dims):\n",
        "        self.n_dims = n_dims\n",
        "\n",
        "    def sample_xs(self):\n",
        "        raise NotImplementedError\n",
        "\n",
        "\n",
        "def get_data_sampler(data_name, n_dims, **kwargs):\n",
        "    names_to_classes = {\n",
        "        \"gaussian\": GaussianSampler,\n",
        "        \"gaussian_centroid\": GaussianCentroidSampler\n",
        "    }\n",
        "    if data_name in names_to_classes:\n",
        "        sampler_cls = names_to_classes[data_name]\n",
        "        return sampler_cls(n_dims, **kwargs)\n",
        "    else:\n",
        "        print(\"Unknown sampler\")\n",
        "        raise NotImplementedError\n",
        "\n",
        "\n",
        "def sample_transformation(eigenvalues, normalize=False):\n",
        "    n_dims = len(eigenvalues)\n",
        "    U, _, _ = torch.linalg.svd(torch.randn(n_dims, n_dims))\n",
        "    t = U @ torch.diag(eigenvalues) @ torch.transpose(U, 0, 1)\n",
        "    if normalize:\n",
        "        norm_subspace = torch.sum(eigenvalues**2)\n",
        "        t *= math.sqrt(n_dims / norm_subspace)\n",
        "    return t\n",
        "\n",
        "\n",
        "class GaussianSampler(DataSampler):  # collect data and split into clusters using KMeans\n",
        "    def __init__(self, n_dims, bias=None, scale=None):\n",
        "        super().__init__(n_dims)\n",
        "        self.bias = bias\n",
        "        self.scale = scale\n",
        "\n",
        "    def sample_xs(self, n_points, b_size, n_dims_truncated=None, seeds=None):\n",
        "        if seeds is None:\n",
        "            xs_b = torch.randn(b_size, n_points, self.n_dims)\n",
        "        else:\n",
        "            xs_b = torch.zeros(b_size, n_points, self.n_dims)\n",
        "            generator = torch.Generator()\n",
        "            assert len(seeds) == b_size\n",
        "            for i, seed in enumerate(seeds):\n",
        "                generator.manual_seed(seed)\n",
        "                xs_b[i] = torch.randn(n_points, self.n_dims, generator=generator)\n",
        "        if self.scale is not None:\n",
        "            xs_b = xs_b @ self.scale\n",
        "        if self.bias is not None:\n",
        "            xs_b += self.bias\n",
        "        if n_dims_truncated is not None:\n",
        "            xs_b[:, :, n_dims_truncated:] = 0\n",
        "\n",
        "        return xs_b\n",
        "\n",
        "class GaussianCentroidSampler(DataSampler):   # generate centroids and generate data near them\n",
        "    labels = {}\n",
        "    def __init__(self, n_dims, bias=None, scale=None):\n",
        "        super().__init__(n_dims)\n",
        "        self.bias = bias\n",
        "        self.scale = .1\n",
        "        self.num_classes = args.task_kwargs[\"classes\"]\n",
        "        self.centroids = torch.randn(self.num_classes, self.n_dims) # get rng distribution centered here\n",
        "\n",
        "    def sample_xs(self, n_points, b_size, n_dims_truncated=None, seeds=None):\n",
        "        xs_b = torch.randn(b_size, n_points, self.n_dims)\n",
        "        for i in range(b_size):\n",
        "          for j in range(n_points):\n",
        "            r = random.randint(0, self.num_classes)\n",
        "            xs_b[i,j] = xs_b[i,j]*self.scale + self.centroids[r]\n",
        "\n",
        "            self.labels[xs_b[i,j]] = r\n",
        "        return xs_b"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xo9Lg1DMl4pk"
      },
      "source": [
        "#Tasks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "GVy1xbStEUmV"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "\n",
        "import torch\n",
        "\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.neighbors import KNeighborsClassifier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "XL4LBg3_l7GA"
      },
      "outputs": [],
      "source": [
        "def squared_error(ys_pred, ys):\n",
        "    return (ys - ys_pred).square()\n",
        "\n",
        "\n",
        "def mean_squared_error(ys_pred, ys):\n",
        "    return (ys - ys_pred).square().mean()\n",
        "\n",
        "\n",
        "def accuracy(ys_pred, ys):\n",
        "    return (ys == ys_pred.sign()).float()\n",
        "\n",
        "\n",
        "sigmoid = torch.nn.Sigmoid()\n",
        "ce_loss = torch.nn.CrossEntropyLoss()\n",
        "\n",
        "\n",
        "def cross_entropy(ys_pred, ys):\n",
        "    output = sigmoid(ys_pred)\n",
        "    target = (ys + 1) / 2\n",
        "    # print(output.shape, target.shape)\n",
        "    return ce_loss(output, target)\n",
        "\n",
        "\n",
        "class Task:\n",
        "    def __init__(self, n_dims, batch_size, pool_dict=None, seeds=None):\n",
        "        self.n_dims = n_dims\n",
        "        self.b_size = batch_size\n",
        "        self.pool_dict = pool_dict\n",
        "        self.seeds = seeds\n",
        "        assert pool_dict is None or seeds is None\n",
        "\n",
        "    def evaluate(self, xs):\n",
        "        raise NotImplementedError\n",
        "\n",
        "    @staticmethod\n",
        "    def generate_pool_dict(n_dims, num_tasks):\n",
        "        raise NotImplementedError\n",
        "\n",
        "    @staticmethod\n",
        "    def get_metric():\n",
        "        raise NotImplementedError\n",
        "\n",
        "    @staticmethod\n",
        "    def get_training_metric():\n",
        "        raise NotImplementedError\n",
        "\n",
        "\n",
        "def get_task_sampler(\n",
        "    task_name, n_dims, batch_size, pool_dict=None, num_tasks=None, **kwargs\n",
        "):\n",
        "    task_names_to_classes = {\n",
        "        \"knn_classification\": KNNClassification,\n",
        "        \"knn_regression\": KNNRegression,\n",
        "    }\n",
        "    if task_name in task_names_to_classes:\n",
        "        task_cls = task_names_to_classes[task_name]\n",
        "        if num_tasks is not None:\n",
        "            if pool_dict is not None:\n",
        "                raise ValueError(\"Either pool_dict or num_tasks should be None.\")\n",
        "            pool_dict = task_cls.generate_pool_dict(n_dims, num_tasks, **kwargs)\n",
        "        return lambda **args: task_cls(n_dims, batch_size, pool_dict, **args, **kwargs)\n",
        "    else:\n",
        "        print(\"Unknown task\")\n",
        "        raise NotImplementedError\n",
        "\n",
        "class KNNClassification(Task):\n",
        "    def __init__(\n",
        "        self,\n",
        "        n_dims,\n",
        "        batch_size,\n",
        "        pool_dict=None,\n",
        "        seeds=None,\n",
        "        k = 2,\n",
        "        num_pregeneration = 100,\n",
        "        num_labels = 5,\n",
        "    ):\n",
        "        super(KNNClassification, self).__init__(n_dims, batch_size, pool_dict, seeds)\n",
        "        self.k = k\n",
        "\n",
        "        values = torch.randn(num_pregeneration, n_dims)\n",
        "        self.kmeans = KMeans(n_clusters=num_labels)\n",
        "        self.kmeans.fit(values)\n",
        "        self.knn = KNeighborsClassifier(n_neighbors=k)\n",
        "        self.knn.fit(values, self.kmeans.labels_)\n",
        "        # print(self.knn.classes_, self.knn.n_features_in_)\n",
        "\n",
        "\n",
        "    def evaluate(self, xs_b):\n",
        "        # print(xs_b.type())\n",
        "        ys_b = [[self.knn.predict([xs_b[i][j].numpy()]) for j in range(xs_b.shape[1])] for i in range(xs_b.shape[0])]\n",
        "        # print(type(self.knn.predict([xs_b[0][0].numpy()])[0]))\n",
        "        ys_b = np.array(ys_b)\n",
        "        ys_b = torch.tensor(ys_b)\n",
        "        ys_b = ys_b[:,:,0]\n",
        "        ys_b = ys_b.to(torch.float)\n",
        "\n",
        "        # print(torch.tensor(ys_b).shape)\n",
        "        # print(xs_b.shape)\n",
        "        # print(xs_b.shape[0])\n",
        "        # print(ys_b.type())\n",
        "\n",
        "        return ys_b\n",
        "\n",
        "    @staticmethod\n",
        "    def get_metric():\n",
        "        return accuracy\n",
        "\n",
        "    @staticmethod\n",
        "    def get_training_metric():\n",
        "        return cross_entropy\n",
        "\n",
        "from sklearn.neighbors import KNeighborsRegressor\n",
        "\n",
        "class KNNRegression(Task):\n",
        "    def __init__(\n",
        "        self,\n",
        "        n_dims,\n",
        "        batch_size,\n",
        "        pool_dict=None,\n",
        "        seeds=None,\n",
        "        k=2,\n",
        "    ):\n",
        "        super(KNNRegression, self).__init__(n_dims, batch_size, pool_dict, seeds)\n",
        "        self.k = k\n",
        "        self.model = KNeighborsRegressor(n_neighbors=self.k)\n",
        "\n",
        "        if pool_dict is None and seeds is None:\n",
        "            self.train_data = torch.randn(self.b_size, n_dims)\n",
        "            self.train_labels = torch.randn(self.b_size)\n",
        "        elif seeds is not None:\n",
        "            self.train_data = torch.zeros(self.b_size, n_dims)\n",
        "            self.train_labels = torch.zeros(self.b_size)\n",
        "            generator = torch.Generator()\n",
        "            assert len(seeds) == self.b_size\n",
        "            for i, seed in enumerate(seeds):\n",
        "                generator.manual_seed(seed)\n",
        "                self.train_data[i] = torch.randn(n_dims, generator=generator)\n",
        "                self.train_labels[i] = torch.randn(1, generator=generator)\n",
        "        else:\n",
        "            assert \"data\" in pool_dict and \"labels\" in pool_dict\n",
        "            indices = torch.randperm(len(pool_dict[\"data\"]))[:batch_size]\n",
        "            self.train_data = pool_dict[\"data\"][indices]\n",
        "            self.train_labels = pool_dict[\"labels\"][indices]\n",
        "\n",
        "    def evaluate(self, xs_b):\n",
        "        xs_b = xs_b.to(self.train_data.device)\n",
        "        ys_b_k = []\n",
        "\n",
        "        for i in range(xs_b.shape[0]):\n",
        "            xs_i = xs_b[i].unsqueeze(0)  # Take one example at a time\n",
        "            distances, indices = self.model.kneighbors(xs_i)\n",
        "            ys_i = self.train_labels[indices].mean()\n",
        "            ys_b_k.append(ys_i)\n",
        "\n",
        "        ys_b_k = torch.stack(ys_b_k)\n",
        "        return ys_b_k\n",
        "\n",
        "    @staticmethod\n",
        "    def generate_pool_dict(n_dims, num_tasks, **kwargs):\n",
        "        raise NotImplementedError\n",
        "\n",
        "    @staticmethod\n",
        "    def get_metric():\n",
        "        return squared_error\n",
        "\n",
        "    @staticmethod\n",
        "    def get_training_metric():\n",
        "        return mean_squared_error"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iKUFD3nwEbz5"
      },
      "source": [
        "#Curriculum"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "uO5HzxMTEfjn"
      },
      "outputs": [],
      "source": [
        "import math"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "2NveV6WDEbXM"
      },
      "outputs": [],
      "source": [
        "class Curriculum:\n",
        "    def __init__(self, args):\n",
        "        # args.dims and args.points each contain start, end, inc, interval attributes\n",
        "        # inc denotes the change in n_dims,\n",
        "        # this change is done every interval,\n",
        "        # and start/end are the limits of the parameter\n",
        "        self.n_dims_truncated = args.dims.start\n",
        "        self.n_points = args.points.start\n",
        "        self.n_dims_schedule = args.dims\n",
        "        self.n_points_schedule = args.points\n",
        "        self.step_count = 0\n",
        "\n",
        "    def update(self):\n",
        "        self.step_count += 1\n",
        "        self.n_dims_truncated = self.update_var(\n",
        "            self.n_dims_truncated, self.n_dims_schedule\n",
        "        )\n",
        "        self.n_points = self.update_var(self.n_points, self.n_points_schedule)\n",
        "\n",
        "    def update_var(self, var, schedule):\n",
        "        if self.step_count % schedule.interval == 0:\n",
        "            var += schedule.inc\n",
        "\n",
        "        return min(var, schedule.end)\n",
        "\n",
        "\n",
        "# returns the final value of var after applying curriculum.\n",
        "def get_final_var(init_var, total_steps, inc, n_steps, lim):\n",
        "    final_var = init_var + math.floor((total_steps) / n_steps) * inc\n",
        "\n",
        "    return min(final_var, lim)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YoTpHYaZE5OX"
      },
      "source": [
        "#Evaluation Metrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "v0AFQsESE7fV"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import os\n",
        "import sys\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "import torch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "QhjsW5C6E_GC"
      },
      "outputs": [],
      "source": [
        "def get_model_from_run(run_path, step=-1, only_conf=False):\n",
        "    conf = args\n",
        "    if only_conf:\n",
        "        return None, conf\n",
        "\n",
        "    model = build_model(conf.model)\n",
        "\n",
        "    if step == -1:\n",
        "        state_path = os.path.join(run_path, \"state.pt\")\n",
        "        state = torch.load(state_path)\n",
        "        model.load_state_dict(state[\"model_state_dict\"])\n",
        "    else:\n",
        "        model_path = os.path.join(run_path, f\"model_{step}.pt\")\n",
        "        state_dict = torch.load(model_path)\n",
        "        model.load_state_dict(state_dict)\n",
        "\n",
        "    return model, conf\n",
        "\n",
        "\n",
        "# Functions for evaluation\n",
        "\n",
        "\n",
        "def eval_batch(model, task_sampler, xs, xs_p=None):\n",
        "    task = task_sampler()\n",
        "    if torch.cuda.is_available() and model.name.split(\"_\")[0] in [\"gpt2\", \"lstm\"]:\n",
        "        device = \"cuda\"\n",
        "    else:\n",
        "        device = \"cpu\"\n",
        "\n",
        "    if xs_p is None:\n",
        "        ys = task.evaluate(xs)\n",
        "        pred = model(xs.to(device), ys.to(device)).detach()\n",
        "        metrics = task.get_metric()(pred.cpu(), ys)\n",
        "    else:\n",
        "        b_size, n_points, _ = xs.shape\n",
        "        metrics = torch.zeros(b_size, n_points)\n",
        "        for i in range(n_points):\n",
        "            xs_comb = torch.cat((xs[:, :i, :], xs_p[:, i:, :]), dim=1)\n",
        "            ys = task.evaluate(xs_comb)\n",
        "\n",
        "            pred = model(xs_comb.to(device), ys.to(device), inds=[i]).detach()\n",
        "            metrics[:, i] = task.get_metric()(pred.cpu(), ys)[:, i]\n",
        "\n",
        "    return metrics\n",
        "\n",
        "\n",
        "# Functions for generating different kinds of train/test data\n",
        "\n",
        "\n",
        "def gen_standard(data_sampler, n_points, b_size):\n",
        "    xs = data_sampler.sample_xs(n_points, b_size)\n",
        "\n",
        "    return xs, None\n",
        "\n",
        "\n",
        "def gen_opposite_quadrants(data_sampler, n_points, b_size):\n",
        "    xs = data_sampler.sample_xs(n_points, b_size)\n",
        "    pattern = torch.randn([b_size, 1, xs.shape[2]]).sign()\n",
        "\n",
        "    xs_train_pre = xs.abs() * pattern\n",
        "    xs_test_post = -xs_train_pre\n",
        "\n",
        "    return xs_train_pre, xs_test_post\n",
        "\n",
        "\n",
        "def gen_random_quadrants(data_sampler, n_points, b_size):\n",
        "    xs = data_sampler.sample_xs(n_points, b_size)\n",
        "    pattern = torch.randn([b_size, 1, xs.shape[2]]).sign()\n",
        "\n",
        "    xs_train_pre = xs.abs() * pattern\n",
        "    xs_test_post = xs\n",
        "\n",
        "    return xs_train_pre, xs_test_post\n",
        "\n",
        "\n",
        "def gen_orthogonal_train_test(data_sampler, n_points, b_size):\n",
        "    xs = data_sampler.sample_xs(n_points, b_size)\n",
        "    n_dim = xs.shape[2]\n",
        "    n_points = min(n_points, n_dim)\n",
        "    # raise ValueError(\"number of points should be at most the dimension.\")\n",
        "    xs_train_pre = xs\n",
        "    xs_test_post = torch.zeros(xs.shape)\n",
        "    for i in range(n_points):\n",
        "        xs_test_post_i = xs[:, i : i + 1, :]\n",
        "        xs_train_pre_i = xs[:, :i, :]\n",
        "        _, _, Vt = torch.linalg.svd(xs_train_pre_i, full_matrices=False)\n",
        "        xs_train_pre_i_projection = Vt.transpose(1, 2) @ Vt\n",
        "        xs_test_post_i_orthogonalized = (\n",
        "            xs_test_post_i - xs_test_post_i @ xs_train_pre_i_projection\n",
        "        )\n",
        "        xs_test_post_i_normalized = (\n",
        "            xs_test_post_i_orthogonalized\n",
        "            * xs_test_post_i.norm(dim=2).unsqueeze(2)\n",
        "            / xs_test_post_i_orthogonalized.norm(dim=2).unsqueeze(2)\n",
        "        )\n",
        "\n",
        "        xs_test_post[:, i : i + 1, :] = xs_test_post_i_normalized\n",
        "\n",
        "    return xs_train_pre, xs_test_post\n",
        "\n",
        "\n",
        "def gen_overlapping_train_test(data_sampler, n_points, b_size):\n",
        "    xs = data_sampler.sample_xs(n_points, b_size)\n",
        "    xs_train_pre = xs\n",
        "    xs_test_post = xs.clone()\n",
        "    b_size = xs.shape[0]\n",
        "    for i in range(1, n_points):\n",
        "        xs_train_pre_i = xs[:, :i, :]\n",
        "        perm = torch.stack([torch.randperm(i) for _ in range(b_size)]).unsqueeze(dim=1)\n",
        "        ind_mat = (perm == 0) + 0.0\n",
        "        xs_test_post[:, i : i + 1, :] = ind_mat @ xs_train_pre_i\n",
        "\n",
        "    return xs_train_pre, xs_test_post\n",
        "\n",
        "\n",
        "def aggregate_metrics(metrics, bootstrap_trials=1000):\n",
        "    \"\"\"\n",
        "    Takes as input a tensor of shape (num_eval, n_points) and returns a dict with\n",
        "    per-point mean, stddev, and bootstrap limits\n",
        "    \"\"\"\n",
        "    results = {}\n",
        "    results[\"mean\"] = metrics.mean(dim=0)\n",
        "    results[\"std\"] = metrics.std(dim=0, unbiased=True)\n",
        "    n = len(metrics)\n",
        "    bootstrap_indices = torch.randint(n, size=(bootstrap_trials, n))\n",
        "    bootstrap_means = metrics[bootstrap_indices].mean(dim=1).sort(dim=0)[0]\n",
        "    results[\"bootstrap_low\"] = bootstrap_means[int(0.05 * bootstrap_trials), :]\n",
        "    results[\"bootstrap_high\"] = bootstrap_means[int(0.95 * bootstrap_trials), :]\n",
        "\n",
        "    return {k: v.tolist() for k, v in results.items()}\n",
        "\n",
        "\n",
        "def eval_model(\n",
        "    model,\n",
        "    task_name,\n",
        "    data_name,\n",
        "    n_dims,\n",
        "    n_points,\n",
        "    prompting_strategy,\n",
        "    num_eval_examples=1280,\n",
        "    batch_size=64,\n",
        "    data_sampler_kwargs={},\n",
        "    task_sampler_kwargs={},\n",
        "):\n",
        "    \"\"\"\n",
        "    Evaluate a model on a task with a variety of strategies.\n",
        "       Args:\n",
        "       - task: which base task we are evaluating on. E.g., \"linear_regression\"\n",
        "       - prompting_strategy: how to construct the prompt, e.g., \"random_quadrants\"\n",
        "       - num_eval_examples: total number of examples to evaluate on\n",
        "       - **sampler_kwargs: remaining arguments to pass directly to the sampler\n",
        "    \"\"\"\n",
        "\n",
        "    assert num_eval_examples % batch_size == 0\n",
        "    data_sampler = get_data_sampler(data_name, n_dims, **data_sampler_kwargs)\n",
        "    task_sampler = get_task_sampler(\n",
        "        task_name, n_dims, batch_size, **task_sampler_kwargs\n",
        "    )\n",
        "\n",
        "    all_metrics = []\n",
        "\n",
        "    generating_func = globals()[f\"gen_{prompting_strategy}\"]\n",
        "    for i in range(num_eval_examples // batch_size):\n",
        "        xs, xs_p = generating_func(data_sampler, n_points, batch_size)\n",
        "\n",
        "        metrics = eval_batch(model, task_sampler, xs, xs_p)\n",
        "        all_metrics.append(metrics)\n",
        "\n",
        "    metrics = torch.cat(all_metrics, dim=0)\n",
        "\n",
        "    return aggregate_metrics(metrics)\n",
        "\n",
        "\n",
        "def build_evals(conf):\n",
        "    n_dims = conf.model.n_dims\n",
        "    n_points = conf.training.curriculum.points.end\n",
        "    batch_size = conf.training.batch_size\n",
        "\n",
        "    task_name = conf.training.task\n",
        "    data_name = conf.training.data\n",
        "\n",
        "    base_kwargs = {\n",
        "        \"task_name\": task_name,\n",
        "        \"n_dims\": n_dims,\n",
        "        \"n_points\": n_points,\n",
        "        \"batch_size\": batch_size,\n",
        "        \"data_name\": data_name,\n",
        "        \"prompting_strategy\": \"standard\",\n",
        "    }\n",
        "\n",
        "    evaluation_kwargs = {}\n",
        "\n",
        "    evaluation_kwargs[\"standard\"] = {\"prompting_strategy\": \"standard\"}\n",
        "    if task_name != \"linear_regression\":\n",
        "        if task_name in [\"relu_2nn_regression\"]:\n",
        "            evaluation_kwargs[\"linear_regression\"] = {\"task_name\": \"linear_regression\"}\n",
        "        for name, kwargs in evaluation_kwargs.items():\n",
        "            # allow kwargs to override base_kwargs values\n",
        "            evaluation_kwargs[name] = base_kwargs.copy()\n",
        "            evaluation_kwargs[name].update(kwargs)\n",
        "        return evaluation_kwargs\n",
        "\n",
        "    for strategy in [\n",
        "        \"random_quadrants\",\n",
        "        \"orthogonal_train_test\",\n",
        "        \"overlapping_train_test\",\n",
        "    ]:\n",
        "        evaluation_kwargs[strategy] = {\"prompting_strategy\": strategy}\n",
        "\n",
        "    for method in [\"half_subspace\", \"skewed\"]:\n",
        "        if \"subspace\" in method:\n",
        "            eigenvals = torch.zeros(n_dims)\n",
        "            eigenvals[: n_dims // 2] = 1\n",
        "        else:\n",
        "            eigenvals = 1 / (torch.arange(n_dims) + 1)\n",
        "\n",
        "        scale = sample_transformation(eigenvals, normalize=True)\n",
        "        evaluation_kwargs[f\"{method}\"] = {\n",
        "            \"data_sampler_kwargs\": {\"scale\": scale},\n",
        "        }\n",
        "\n",
        "    for dim in [\"x\", \"y\"]:\n",
        "        for scale in [0.333, 0.5, 2, 3]:\n",
        "            if dim == \"x\":\n",
        "                eigenvals = scale * torch.ones(n_dims)\n",
        "                t = sample_transformation(eigenvals)\n",
        "                scaling_args = {\"data_sampler_kwargs\": {\"scale\": t}}\n",
        "            else:\n",
        "                eigenvals = scale * torch.ones(n_dims)\n",
        "                scaling_args = {\"task_sampler_kwargs\": {\"scale\": scale}}\n",
        "\n",
        "            evaluation_kwargs[f\"scale-{dim}={scale}\"] = scaling_args\n",
        "\n",
        "    evaluation_kwargs[f\"noisyLR\"] = {\n",
        "        \"task_sampler_kwargs\": {\"renormalize_ys\": True, \"noise_std\": 1},\n",
        "        \"task_name\": \"noisy_linear_regression\",\n",
        "    }\n",
        "\n",
        "    for name, kwargs in evaluation_kwargs.items():\n",
        "        # allow kwargs to override base_kwargs values\n",
        "        evaluation_kwargs[name] = base_kwargs.copy()\n",
        "        evaluation_kwargs[name].update(kwargs)\n",
        "\n",
        "    return evaluation_kwargs\n",
        "\n",
        "\n",
        "def compute_evals(all_models, evaluation_kwargs, save_path=None, recompute=False):\n",
        "    try:\n",
        "        with open(save_path) as fp:\n",
        "            all_metrics = json.load(fp)\n",
        "    except Exception:\n",
        "        all_metrics = {}\n",
        "\n",
        "    for eval_name, kwargs in tqdm(evaluation_kwargs.items()):\n",
        "        metrics = {}\n",
        "        if eval_name in all_metrics and not recompute:\n",
        "            metrics = all_metrics[eval_name]\n",
        "        for model in all_models:\n",
        "            if model.name in metrics and not recompute:\n",
        "                continue\n",
        "\n",
        "            metrics[model.name] = eval_model(model, **kwargs)\n",
        "        all_metrics[eval_name] = metrics\n",
        "\n",
        "    if save_path is not None:\n",
        "        with open(save_path, \"w\") as fp:\n",
        "            json.dump(all_metrics, fp, indent=2)\n",
        "\n",
        "    return all_metrics\n",
        "\n",
        "\n",
        "def get_run_metrics(\n",
        "    run_path, step=-1, cache=True, skip_model_load=False, skip_baselines=False\n",
        "):\n",
        "    if skip_model_load:\n",
        "        _, conf = get_model_from_run(run_path, only_conf=True)\n",
        "        all_models = []\n",
        "    else:\n",
        "        model, conf = get_model_from_run(run_path, step)\n",
        "        model = model.cuda().eval()\n",
        "        all_models = [model]\n",
        "        if not skip_baselines:\n",
        "            all_models += get_relevant_baselines(conf.training.task)\n",
        "    evaluation_kwargs = build_evals(conf)\n",
        "\n",
        "    if not cache:\n",
        "        save_path = None\n",
        "    elif step == -1:\n",
        "        save_path = os.path.join(run_path, \"metrics.json\")\n",
        "    else:\n",
        "        save_path = os.path.join(run_path, f\"metrics_{step}.json\")\n",
        "\n",
        "    recompute = False\n",
        "    if save_path is not None and os.path.exists(save_path):\n",
        "        checkpoint_created = os.path.getmtime(run_path)\n",
        "        cache_created = os.path.getmtime(save_path)\n",
        "        if checkpoint_created > cache_created:\n",
        "            recompute = True\n",
        "\n",
        "    all_metrics = compute_evals(all_models, evaluation_kwargs, save_path, recompute)\n",
        "    return all_metrics\n",
        "\n",
        "\n",
        "\n",
        "def conf_to_model_name(conf):\n",
        "    if conf.model.family == \"gpt2\":\n",
        "        return {\n",
        "            (3, 2): \"Transformer-xs\",\n",
        "            (6, 4): \"Transformer-small\",\n",
        "            (12, 8): \"Transformer\",\n",
        "        }[(conf.model.n_layer, conf.model.n_head)]\n",
        "    else:\n",
        "        return conf.wandb.name\n",
        "\n",
        "\n",
        "def baseline_names(name):\n",
        "    if \"OLS\" in name:\n",
        "        return \"Least Squares\"\n",
        "    if name == \"averaging\":\n",
        "        return \"Averaging\"\n",
        "    if \"NN\" in name:\n",
        "        k = name.split(\"_\")[1].split(\"=\")[1]\n",
        "        return f\"{k}-Nearest Neighbors\"\n",
        "    if \"lasso\" in name:\n",
        "        alpha = name.split(\"_\")[1].split(\"=\")[1]\n",
        "        return f\"Lasso (alpha={alpha})\"\n",
        "    if \"gd\" in name:\n",
        "        return \"2-layer NN, GD\"\n",
        "    if \"decision_tree\" in name:\n",
        "        return \"Greedy Tree Learning\"\n",
        "    if \"xgboost\" in name:\n",
        "        return \"XGBoost\"\n",
        "    return name\n",
        "\n",
        "\n",
        "def read_run_dir(run_dir):\n",
        "    all_runs = {}\n",
        "    for task in os.listdir(run_dir):\n",
        "        task_dir = os.path.join(run_dir, task)\n",
        "        for run_id in os.listdir(task_dir):\n",
        "            run_path = os.path.join(task_dir, run_id)\n",
        "            _, conf = get_model_from_run(run_path, only_conf=True)\n",
        "            params = {}\n",
        "            params[\"run_id\"] = run_id\n",
        "            params[\"task\"] = task\n",
        "            params[\"model\"] = conf_to_model_name(conf)\n",
        "            params[\"kwargs\"] = \"_\".join(\n",
        "                f\"{k}={v}\" for k, v in conf.training.task_kwargs.items()\n",
        "            )\n",
        "            num_tasks = (\n",
        "                conf.training.num_tasks if \"num_tasks\" in conf.training else None\n",
        "            )\n",
        "            params[\"num_tasks\"] = num_tasks if num_tasks is not None else -1\n",
        "            num_examples = (\n",
        "                conf.training.num_training_examples\n",
        "                if \"num_training_examples\" in conf.training\n",
        "                else None\n",
        "            )\n",
        "            params[\"num_examples\"] = num_examples if num_examples is not None else -1\n",
        "            params[\"n_dims\"] = conf.model.n_dims\n",
        "            params[\"n_layer\"] = conf.model.n_layer\n",
        "            params[\"n_head\"] = conf.model.n_head\n",
        "            params[\"run_name\"] = conf.wandb.name\n",
        "\n",
        "            for k, v in params.items():\n",
        "                if k not in all_runs:\n",
        "                    all_runs[k] = []\n",
        "                all_runs[k].append(v)\n",
        "\n",
        "    df = pd.DataFrame(all_runs).sort_values(\"run_name\")\n",
        "    assert len(df) == len(df.run_name.unique())\n",
        "    return df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ByhU5udsu8o0"
      },
      "source": [
        "#Train\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "aScQG1VoDMMu"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "from random import randint\n",
        "import uuid\n",
        "\n",
        "from tqdm import tqdm\n",
        "import torch\n",
        "\n",
        "import wandb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "8M4Uj024u9PG"
      },
      "outputs": [],
      "source": [
        "torch.backends.cudnn.benchmark = True\n",
        "\n",
        "\n",
        "def train_step(model, xs, ys, optimizer, loss_func):\n",
        "    optimizer.zero_grad()\n",
        "    output = model(xs, ys)\n",
        "    loss = loss_func(output, ys)\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    return loss.detach().item(), output.detach()\n",
        "\n",
        "\n",
        "def sample_seeds(total_seeds, count):\n",
        "    seeds = set()\n",
        "    while len(seeds) < count:\n",
        "        seeds.add(randint(0, total_seeds - 1))\n",
        "    return seeds\n",
        "\n",
        "\n",
        "def train(model, args):\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=args.training.learning_rate)\n",
        "    curriculum = Curriculum(args.training.curriculum)\n",
        "\n",
        "    starting_step = 0\n",
        "    state_path = os.path.join(args.out_dir, \"state.pt\")\n",
        "    if os.path.exists(state_path):\n",
        "        state = torch.load(state_path)\n",
        "        model.load_state_dict(state[\"model_state_dict\"])\n",
        "        optimizer.load_state_dict(state[\"optimizer_state_dict\"])\n",
        "        starting_step = state[\"train_step\"]\n",
        "        for i in range(state[\"train_step\"] + 1):\n",
        "            curriculum.update()\n",
        "\n",
        "    n_dims = model.n_dims\n",
        "    bsize = args.training.batch_size\n",
        "    data_sampler = get_data_sampler(args.training.data, n_dims=n_dims)\n",
        "    task_sampler = get_task_sampler(\n",
        "        args.training.task,\n",
        "        n_dims,\n",
        "        bsize,\n",
        "        num_tasks=args.training.num_tasks,\n",
        "        **args.training.task_kwargs,\n",
        "    )\n",
        "    pbar = tqdm(range(starting_step, args.training.train_steps))\n",
        "\n",
        "    num_training_examples = args.training.num_training_examples\n",
        "\n",
        "    for i in pbar:\n",
        "        data_sampler_args = {}\n",
        "        task_sampler_args = {}\n",
        "\n",
        "        if \"sparse\" in args.training.task:\n",
        "            task_sampler_args[\"valid_coords\"] = curriculum.n_dims_truncated\n",
        "        if num_training_examples is not None:\n",
        "            assert num_training_examples >= bsize\n",
        "            seeds = sample_seeds(num_training_examples, bsize)\n",
        "            data_sampler_args[\"seeds\"] = seeds\n",
        "            task_sampler_args[\"seeds\"] = [s + 1 for s in seeds]\n",
        "\n",
        "        xs = data_sampler.sample_xs(\n",
        "            curriculum.n_points,\n",
        "            bsize,\n",
        "            curriculum.n_dims_truncated,\n",
        "            **data_sampler_args,\n",
        "        )\n",
        "        task = task_sampler(**task_sampler_args)\n",
        "        ys = task.evaluate(xs)\n",
        "\n",
        "        loss_func = task.get_training_metric()\n",
        "\n",
        "        loss, output = train_step(model, xs.cuda(), ys.cuda(), optimizer, loss_func)\n",
        "\n",
        "        point_wise_tags = list(range(curriculum.n_points))\n",
        "        point_wise_loss_func = task.get_metric()\n",
        "        point_wise_loss = point_wise_loss_func(output, ys.cuda()).mean(dim=0)\n",
        "\n",
        "        baseline_loss = (\n",
        "            sum(\n",
        "                max(curriculum.n_dims_truncated - ii, 0)\n",
        "                for ii in range(curriculum.n_points)\n",
        "            )\n",
        "            / curriculum.n_points\n",
        "        )\n",
        "\n",
        "        if i % args.wandb.log_every_steps == 0 and not args.test_run and args.using_wandb:\n",
        "            wandb.log(\n",
        "                {\n",
        "                    \"overall_loss\": loss,\n",
        "                    \"excess_loss\": loss / baseline_loss,\n",
        "                    \"pointwise/loss\": dict(\n",
        "                        zip(point_wise_tags, point_wise_loss.cpu().numpy())\n",
        "                    ),\n",
        "                    \"n_points\": curriculum.n_points,\n",
        "                    \"n_dims\": curriculum.n_dims_truncated,\n",
        "                },\n",
        "                step=i,\n",
        "            )\n",
        "\n",
        "        curriculum.update()\n",
        "\n",
        "        pbar.set_description(f\"loss {loss}\")\n",
        "        if i % args.training.save_every_steps == 0 and not args.test_run:\n",
        "            training_state = {\n",
        "                \"model_state_dict\": model.state_dict(),\n",
        "                \"optimizer_state_dict\": optimizer.state_dict(),\n",
        "                \"train_step\": i,\n",
        "            }\n",
        "            torch.save(training_state, state_path)\n",
        "\n",
        "        if (\n",
        "            args.training.keep_every_steps > 0\n",
        "            and i % args.training.keep_every_steps == 0\n",
        "            and not args.test_run\n",
        "            and i > 0\n",
        "        ):\n",
        "            torch.save(model.state_dict(), os.path.join(args.out_dir, f\"model_{i}.pt\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vQrcM2wIwgZg",
        "outputId": "3c8aafe3-7a9d-4048-914e-3668c1b851b3"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loss 136.1259307861328: 100%|██████████| 5001/5001 [1:49:43<00:00,  1.32s/it]\n",
            "100%|██████████| 1/1 [03:55<00:00, 235.89s/it]\n"
          ]
        }
      ],
      "source": [
        "if args.using_wandb:\n",
        "    wandb.init(\n",
        "        dir=args.out_dir,\n",
        "        project=args.wandb.project,\n",
        "        entity=args.wandb.entity,\n",
        "        config=args.__dict__,\n",
        "        notes=args.wandb.notes,\n",
        "        name=args.wandb.name,\n",
        "        resume=True,\n",
        "    )\n",
        "\n",
        "run_id = args.training.resume_id\n",
        "if run_id is None:\n",
        "    run_id = str(uuid.uuid4())\n",
        "\n",
        "out_dir = os.path.join(args.out_dir, run_id)\n",
        "if not os.path.exists(out_dir):\n",
        "    os.makedirs(out_dir)\n",
        "args.out_dir = out_dir\n",
        "\n",
        "model = build_model(args.model)\n",
        "model.cuda()\n",
        "model.train()\n",
        "\n",
        "train(model, args)\n",
        "\n",
        "_ = get_run_metrics(args.out_dir)  # precompute metrics for eval\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ANEUBJNpYOr"
      },
      "source": [
        "#Evaluate on new data\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SSby9hviLhEw"
      },
      "source": [
        "Plot Utils\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "7AcnzjkGLjaX"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "iTmye-K5LifU"
      },
      "outputs": [],
      "source": [
        "sns.set_theme(\"notebook\", \"darkgrid\")\n",
        "palette = sns.color_palette(\"colorblind\")\n",
        "\n",
        "\n",
        "relevant_model_names = {\n",
        "    \"knn_classification\": [\n",
        "        \"Transformer\",\n",
        "        \"3-Nearest Neighbors\",\n",
        "        \"Averaging\",\n",
        "    ]\n",
        "}\n",
        "\n",
        "\n",
        "def basic_plot(metrics, models=None, trivial=1.0):\n",
        "    fig, ax = plt.subplots(1, 1)\n",
        "\n",
        "    if models is not None:\n",
        "        metrics = {k: metrics[k] for k in models}\n",
        "\n",
        "    color = 0\n",
        "    ax.axhline(trivial, ls=\"--\", color=\"gray\")\n",
        "    for name, vs in metrics.items():\n",
        "        ax.plot(vs[\"mean\"], \"-\", label=name, color=palette[color % 10], lw=2)\n",
        "        low = vs[\"bootstrap_low\"]\n",
        "        high = vs[\"bootstrap_high\"]\n",
        "        ax.fill_between(range(len(low)), low, high, alpha=0.3)\n",
        "        color += 1\n",
        "    ax.set_xlabel(\"in-context examples\")\n",
        "    ax.set_ylabel(\"squared error\")\n",
        "    ax.set_xlim(-1, len(low) + 0.1)\n",
        "    ax.set_ylim(-0.1, 6)\n",
        "\n",
        "    legend = ax.legend(loc=\"upper left\", bbox_to_anchor=(1, 1))\n",
        "    fig.set_size_inches(4, 3)\n",
        "    for line in legend.get_lines():\n",
        "        line.set_linewidth(3)\n",
        "\n",
        "    return fig, ax\n",
        "\n",
        "\n",
        "def collect_results(run_path, task):\n",
        "    all_metrics = {}\n",
        "\n",
        "    _, conf = get_model_from_run(run_path, only_conf=True)\n",
        "\n",
        "    print(run_path)\n",
        "    metrics = get_run_metrics(run_path, skip_model_load=True)\n",
        "\n",
        "    for eval_name, results in sorted(metrics.items()):\n",
        "        processed_results = {}\n",
        "        for model_name, m in results.items():\n",
        "            if \"gpt2\" in model_name in model_name:\n",
        "                model_name = \"Transformer\"\n",
        "            else:\n",
        "                model_name = baseline_names(model_name)\n",
        "            m_processed = {}\n",
        "            n_dims = conf.model.n_dims\n",
        "\n",
        "            xlim = 2 * n_dims + 1\n",
        "            if task in [\"relu_2nn_regression\", \"decision_tree\"]:\n",
        "                xlim = 200\n",
        "\n",
        "            normalization = n_dims\n",
        "            if task == \"sparse_linear_regression\":\n",
        "                normalization = int(args.training.task_kwargs.split(\"=\")[-1])\n",
        "            if task == \"decision_tree\":\n",
        "                normalization = 1\n",
        "\n",
        "            for k, v in m.items():\n",
        "                v = v[:xlim]\n",
        "                v = [vv / normalization for vv in v]\n",
        "                m_processed[k] = v\n",
        "            processed_results[model_name] = m_processed\n",
        "        if eval_name not in all_metrics:\n",
        "            all_metrics[eval_name] = {}\n",
        "        all_metrics[eval_name].update(processed_results)\n",
        "    return all_metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9eVIiFOzLog4"
      },
      "source": [
        "Evaluate Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "pA_FGE25pY0F"
      },
      "outputs": [],
      "source": [
        "from collections import OrderedDict\n",
        "import re\n",
        "import os\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import torch\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "\n",
        "%matplotlib inline\n",
        "%load_ext autoreload\n",
        "%autoreload 2\n",
        "\n",
        "sns.set_theme('notebook', 'darkgrid')\n",
        "palette = sns.color_palette('colorblind')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "gmKsIVZjpeN5"
      },
      "outputs": [],
      "source": [
        "task = args.training.task\n",
        "\n",
        "# run_path leads to the directory which holds metrics.json and state.pt\n",
        "# peer reviewer can just change this and we can upload our compressed data somewhere\n",
        "# very low runtime for the rest of the cells\n",
        "run_path = args.out_dir\n",
        "recompute_metrics = False\n",
        "\n",
        "if recompute_metrics:\n",
        "    get_run_metrics(run_path)  # these are normally precomputed at the end of training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6OBV59Dppg2c"
      },
      "source": [
        "Plot precomputed metrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 431,
          "referenced_widgets": [
            "cd30fdbadb2a4944b24e85c9bb1fd98e",
            "d619083e4d4549c48d5a20c25f72fe30",
            "bc60d092b6314b93bf564a7fb327d246",
            "714f5667409f4901bde3532d57d75b81",
            "6c61003c168c4b17a26c2e4158fcc2c5",
            "6d509a1b559441b38d0128658c9aa3d7",
            "ac74f331214c46ed8d05d649f3e6e49d",
            "7653471e6e0a45ba9bc01ee2e79c3e8d",
            "83aceeb438b04ecc82efb30237f0f586",
            "26eaac5185324eff8fb113c2e5950992",
            "80e76cc886a94c6e9b9c5fc223ed9bca"
          ]
        },
        "id": "LRJ2R4lhpfcw",
        "outputId": "22b0a995-e3bf-4812-88b1-c25d191f0a63"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "./models/test/90813c30-3373-4a46-b0f1-0ea56d5dd447\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cd30fdbadb2a4944b24e85c9bb1fd98e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['Transformer', '3-Nearest Neighbors', 'Averaging']\n",
            "{'standard': {'Transformer': {'mean': [0.009232954545454546, 0.008238636634566567, 0.0077414770017970695, 0.006178977137262171, 0.007315341044556011, 0.008451704274524342, 0.007173295725475658, 0.008806817910887978, 0.00788352299820293, 0.007670454680919647, 0.007315341044556011, 0.008806817910887978, 0.0077414770017970695, 0.008522727272727272, 0.007173295725475658, 0.00724431804635308, 0.008309658955443989, 0.007599431682716717, 0.008025568317283283, 0.006392045454545455, 0.007599431682716717, 0.0066761360927061605, 0.008096590638160706], 'std': [0.027471834963018246, 0.02610793709754944, 0.025383924896066837, 0.02289005843075839, 0.02473853122104298, 0.026409284635023636, 0.024517977779561825, 0.026900280605662952, 0.025593860582871872, 0.0252780020236969, 0.02473853122104298, 0.026900280605662952, 0.025383924896066837, 0.026508594101125545, 0.024517977779561825, 0.02462860400026495, 0.02620896426114169, 0.02517143433744257, 0.02580130642110651, 0.023252078078009865, 0.02517143433744257, 0.023723204027522694, 0.025904116305437954], 'bootstrap_low': [0.008096590638160706, 0.0070312497290697966, 0.0066761360927061605, 0.005113636228171262, 0.006178977137262171, 0.00724431804635308, 0.006107954816384749, 0.007599431682716717, 0.006747159090909091, 0.006534090773625808, 0.006250000135465102, 0.007599431682716717, 0.006605113771828738, 0.007386363365433433, 0.006107954816384749, 0.006178977137262171, 0.007102272727272727, 0.006463068452748385, 0.006889204409989444, 0.005326704545454545, 0.006463068452748385, 0.005610795522278006, 0.006960227408192374], 'bootstrap_high': [0.01058238609270616, 0.00944602286273783, 0.00894886390729384, 0.007315341044556011, 0.008451704274524342, 0.009659091180021112, 0.00838068195364692, 0.01008522713726217, 0.009090909226374193, 0.00887784090909091, 0.008522727272727272, 0.010014204816384749, 0.00894886390729384, 0.009801136499101465, 0.00838068195364692, 0.008309658955443989, 0.009517045183615251, 0.008806817910887978, 0.009303977543657476, 0.007528409361839294, 0.008735795590010557, 0.0077414770017970695, 0.009374999864534899]}, 'Least Squares': {'mean': [0.024147727272727272, 0.01775568181818182, 0.013423295183615251, 0.013778408819978888, 0.012002840638160706, 0.013139204545454546, 0.013778408819978888, 0.013139204545454546, 0.012713067910887978, 0.014133522456342524, 0.01164772700179707, 0.01171875, 0.01171875, 0.012286931276321411, 0.01278409090909091, 0.012428977272727272, 0.012286931276321411, 0.012002840638160706, 0.011434659361839294, 0.013281250541860407, 0.010937499729069796, 0.012428977272727272, 0.012073863636363636], 'std': [0.04016709598627957, 0.03605418313633312, 0.0322634144262834, 0.032612405040047386, 0.030787004665894943, 0.031978639689358795, 0.032612405040047386, 0.031978639689358795, 0.0315418541431427, 0.032953871922059494, 0.030396326021714645, 0.03047519380396063, 0.03047519380396063, 0.03109309348193082, 0.03161547129804438, 0.031244042244824497, 0.03109309348193082, 0.030787004665894943, 0.030157479372891514, 0.03212165832519531, 0.029586705294522373, 0.031244042244824497, 0.03086405721577731], 'bootstrap_low': [0.022372159090909092, 0.01626420427452434, 0.011931817639957775, 0.012357954274524342, 0.01065340909090909, 0.01164772700179707, 0.012286931276321411, 0.01178977299820293, 0.011221591044556011, 0.012642044912685047, 0.010369318452748385, 0.010369318452748385, 0.010298295454545454, 0.010866477408192375, 0.011434659361839294, 0.011008522727272728, 0.010866477408192375, 0.01065340909090909, 0.010014204816384749, 0.01178977299820293, 0.009588068181818182, 0.011008522727272728, 0.01065340909090909], 'bootstrap_high': [0.025994317098097366, 0.01953125, 0.014843749729069796, 0.015340909361839294, 0.013494318181818182, 0.01455965909090909, 0.015269886363636364, 0.014630682089112022, 0.014204545454545454, 0.015625, 0.013139204545454546, 0.013068181547251615, 0.013139204545454546, 0.013707385821775957, 0.014346591450951317, 0.013849431818181818, 0.013707385821775957, 0.013352272185412321, 0.01285511390729384, 0.014630682089112022, 0.012357954274524342, 0.013991477814587679, 0.013494318181818182]}, '3-Nearest Neighbors': {'mean': [0.026349430734461003, 0.026988636363636364, 0.02500000054186041, 0.026065341450951317, 0.026633522727272728, 0.025923295454545456, 0.02791193127632141, 0.02578124945813959, 0.025994317098097366, 0.028338069265538997, 0.02563920346173373, 0.024857954545454544, 0.028053977272727272, 0.027698863636363636, 0.028338069265538997, 0.02571022781458768, 0.02720170400359414, 0.028622158549048683, 0.02776988527991555, 0.028124998916279186, 0.02883522618900646, 0.027272728356448086, 0.02500000054186041], 'std': [0.04126064072955738, 0.041550869291478935, 0.040608072822744194, 0.041127800941467285, 0.04139110175046054, 0.041060480204495514, 0.0419493398883126, 0.04099256071177396, 0.041094216433438385, 0.042125149206681686, 0.040924031626094474, 0.04053615440021862, 0.04200850562615828, 0.041859531944448296, 0.042125149206681686, 0.040958372029391205, 0.04164497960697521, 0.04223956032232805, 0.041889610615643585, 0.04203787716952237, 0.042323911731893364, 0.04167606071992354, 0.040608072822744194], 'bootstrap_low': [0.024360794912685047, 0.024857954545454544, 0.02301136472008445, 0.024218748916279186, 0.024857954545454544, 0.02400568127632141, 0.025994317098097366, 0.024076705629175358, 0.024147727272727272, 0.026420455087314953, 0.023650567639957775, 0.02301136472008445, 0.026065341450951317, 0.02578124945813959, 0.026491476730866867, 0.023934659632769497, 0.025284089825370094, 0.026633522727272728, 0.02585227381099354, 0.026207387447357178, 0.02691761472008445, 0.025426135821775955, 0.023153408007188278], 'bootstrap_high': [0.028196023269133133, 0.02890625054186041, 0.02691761472008445, 0.028053977272727272, 0.028480112552642822, 0.027840909632769497, 0.02975852381099354, 0.027556817639957775, 0.027840909632769497, 0.030184659090909092, 0.027556817639957775, 0.026704544370824642, 0.029971591450951317, 0.02961647781458768, 0.030397726730866867, 0.027627841992811722, 0.02911931818181818, 0.03068181872367859, 0.02954545346173373, 0.03004261309450323, 0.030894886363636364, 0.029190339825370094, 0.02677556872367859]}, 'Averaging': {'mean': [0.02975852381099354, 0.018749999729069797, 0.01576704599640586, 0.013352272185412321, 0.01278409090909091, 0.012002840638160706, 0.012073863636363636, 0.01178977299820293, 0.013210227543657476, 0.011221591044556011, 0.01115056804635308, 0.012571023269133135, 0.012215909632769499, 0.010724432089112022, 0.010440340773625807, 0.011079545725475658, 0.012215909632769499, 0.011576704003594139, 0.011505682360042225, 0.009801136499101465, 0.011576704003594139, 0.011292613365433433, 0.01115056804635308], 'std': [0.04267520796168934, 0.03679729320786216, 0.03443391214717518, 0.0321926935152574, 0.03161547129804438, 0.030787004665894943, 0.03086405721577731, 0.030553693121129818, 0.0320503061467951, 0.029915208166295833, 0.029833674430847168, 0.031393622810190376, 0.031017097559842197, 0.029336108402772385, 0.02899617227641019, 0.02975175055590543, 0.031017097559842197, 0.030317084355787796, 0.03023747151548212, 0.028205879709937355, 0.030317084355787796, 0.02999635176225142, 0.029833674430847168], 'bootstrap_low': [0.02791193127632141, 0.017045454545454544, 0.014346591450951317, 0.011931817639957775, 0.011363636363636364, 0.01065340909090909, 0.010724432089112022, 0.010369318452748385, 0.01171875, 0.009872158819978888, 0.009730113500898535, 0.011221591044556011, 0.010724432089112022, 0.009374999864534899, 0.009019886228171263, 0.009659091180021112, 0.010795454409989443, 0.010298295454545454, 0.010156250135465101, 0.008522727272727272, 0.010156250135465101, 0.009943181818181818, 0.009801136499101465], 'bootstrap_high': [0.0317471596327695, 0.020383523269133133, 0.01740056818181818, 0.014914772727272728, 0.014346591450951317, 0.013494318181818182, 0.013423295183615251, 0.013210227543657476, 0.014701705087314953, 0.012642044912685047, 0.012642044912685047, 0.013991477814587679, 0.013636364178224043, 0.012073863636363636, 0.01178977299820293, 0.012500000270930204, 0.013636364178224043, 0.013068181547251615, 0.01292613690549677, 0.01115056804635308, 0.012997158549048683, 0.012642044912685047, 0.012428977272727272]}}}\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlUAAAE0CAYAAAAFRsbQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSeElEQVR4nO3dd1wU1xoG4HcLHZYiiAoSK1hQsYK9a4jdqNiw9xrUCMYWE68xxhJ77yX2LoqxobElGqOJsWOMJaDSO+zu3D+QlZUiLIO7yPvcnzfMzJkz3zmzy37MOTsjEQRBABERERHli1TfARARERF9DJhUEREREYmASRURERGRCJhUEREREYmASRURERGRCJhUEREREYmASRURERGRCJhUEREREYmASRURERGRCAwyqTpw4AA6d+6MatWqwdPTE0OGDEFSUpK+wyIiIiLKllzfAbxr5cqVWLt2LUaMGAEPDw9ERkbi8uXLUKlU+g6NiIiIKFsSQ3r2X0hICDp06IAVK1agadOm+g6HiIiIKNcMavhv//79cHZ2ZkJFREREhY5BJVU3b96Eq6srVqxYgfr168Pd3R09e/bEzZs39R0aERERUY4Mavjv008/RVhYGIoXLw4/Pz+YmZlh1apVuH//Pk6ePIlixYrpVK8gCJBIJCJHS0RERPSWQU1UFwQBCQkJWLx4MSpVqgQAqFGjBlq0aIFt27Zh/PjxOtWrVguIiUnIVVmZTAqFwgwxMYlQqdQ6Ha+wK+p9wPYX7fYDht8HCoUZZDKDGmggIhhYUqVQKGBjY6NJqADAxsYGVapUwcOHD/NVt1KZt1+MKpU6z/t8bIp6H7D9Rbv9APuAiPLGoP7UqVChQrbbkpOTP2AkRERERHljUElV8+bNERUVhTt37mjWRUZG4vbt26hataoeIyMiIiLKmUEN/7Vq1QrVqlXDuHHj4OfnBxMTE6xZswbGxsbo3bu3vsMjIiIiypZBXamSSqVYs2YNPDw8MGPGDEyYMAGWlpbYvn07HBwc9B0eERERUbYM6koVANjZ2eGHH37QdxhERIWeSqVCamqqvsMgKrSMjIwgk8lyXd7gkioiIsofQRDw33//ISoqCoZzJ0KiwkciSbsLQcmSJXN1v0smVUREH5n//vsPkZFRsLKygYmJCQDe/Jgo7wQkJycjMjIKAFCqVKn37sGkiojoI6JSqRAVlZZQWVlZ6zscokLN2NgUABAVFQVHR8f3DgUa1ER1IiLKn9TUVAgC3lyhIqL8MjExgSAgV/MTmVQREX2UOORHJI7cv5eYVBERERGJgEkVERERkQg4UZ2IiAyOl1et95aZNu1rtG/f8QNEk70XL55jzpxv8Pfft5GQkIAtW36Cq6ubXmMi/WFSRUREBmfduk1ay0OGDED37j3Rtu2nmnVOTqU/cFSZrV69As+fP8ecOfNgaWkJFxcXfYdEesSkioiIDI67e/VM60qUKJHl+nRJSUkwNTUtyLAyefLkH3h41ISXV4N816WP+AEgJSUFcrkcUilnBOUXkyoioiJApRYQmai/R9bYmhlBJhXvG4lr167Cjh1bsWzZaixa9APu37+H4cNHoU+ffli+fAkuXfoFL148h6WlJTw8amH8+Amwt3/7DNmRI4fC3NwM7dp1xKpVy/Hq1UtUqVIVU6ZMh7Pz2ytgW7ZsxOHDB/Dy5UuYm5ujQgVXfPXVdAASdO3aHgBw9+4dHD9+DCVKlMTBg8cAAAcO7MVPP23Hf/+9QLFi9ujYsQsGDBikSVyOHj2M2bO/xtq1G7FmzSr8+edNtG/fEc2bt8Lo0cPw44/LcOTIIVy69AsUCmuMGjUWbdt6Y9eun7Bjx1YkJiagadMW+PLLABgbG2viffkyDMuXL8GVK5eRlJSIypWr4IsvJqJSpSqaMp07t0OjRo3h6FgC+/btRlhYGI4fPwUbG1vRzk9RxaSKiOgjd+jOS/gH3cOreP0lVQ4WRvi+rRs6VS4uWp2pqamYOfMr9OzZByNHjoFCkXaz08jICPTvPwj29g6IiorEjh1bMXLkUPz0017I5W8/9u7fv4/IyC0YNWos1Go1Fi9eiK+/noZ16zYDAAIDj2L16pUYNmwE3N2rIz4+Dn/8cQPx8fH45JMyWLduE2bNmoHSpV0waNAQGBmlJTe7d+/EwoXz0L17TzRq1Bi3bt3E+vWrERcXi3Hj/LTaMGPGVHTu3BUDBgyCiYkpkpOTAQDz5n2Hdu06oGPHLjh8+ABmzZqOBw/uIyTkEfz9v8Lz58+wePFCODk5YcCAwQCAmJgYDBs2CObm5pg4cTIsLS2xe/dOjB49Anv2HISdnZ3muGfPnkHp0qXh5/clpFIpTE3NRDsvRRmTKiKij9wXx+4iJlmp1xhexafii2N3RU2qlEolhg8fjdat22qtnzbta83PKpUK7u7V0bHjp7h+/Td4etbXbIuLi8WWLT/B1jbtCk1CQgJmz/4aL1+GoXhxR/z991+oUKEC+vcfpNmnSZNmmp/d3avD1NQUNja2mmFJlUqFDRvWoHXrtpg4cTIAwNOzPpTKVOzYsQ39+w+EtbWNpo4uXbqhX78BmuXr168BAFq0aIXBg4cBAKpWrYpz587g55+DsG/fIcjlRgCA33+/jjNnTmmSql27tiMuLhYbNmzVJFB16tRDjx6dsX37Fowd+4VW3y1atAxmZkymxMQBVCIiKrQaNmycad2lSxcxdOgAtGzZBA0b1kXHjmmT2//994lWuYoV3TQJFQCULVsOQNoQGgC4uVXC/fv38OOPC/DHHzegVL7/St+TJ/8gKioKLVq00lrfqlVbpKam4vbt2+/E3yjLeurV89L8bGlpBVtbW3h41NQkVADg4uKCsLAwzfLVq1dQu3ZdKBQKKJVKKJVKSKVS1KxZG3fu/K1Vf61atZlQFQBeqSIi+sj92K6SwQz/icnU1BTm5uZa6/7++za+/NIPTZo0ha/vQNjZ2QKQYMiQ/khJSdEqa2VlqbVsZJSWsCQnp5Vr164jEhIScPDgfuzcuR2Wlpb47LMOGDVqbLYTymNiYgAAdnbFtNanXzmKiYl+Z712ubexWWkty+VGWa5LSUnWLEdFReGvv/5Eo0b1MtXn7OycZTwkLiZVREQfuU6Vi6O9m8NHNVEdACSSzPUFB5+BpaUl/ve/7zWTwv/774VO9UulUvj49IaPT2+8fPkSP/8chBUrlsLGxgaDBg3Nch9r67fzujKKiEhbTp/39bYNOoWWJYXCGl5epTF8+KhM29ITxrfH5WOMCgKTKiKiIkAmlcDewvj9BQu55ORkyOVyraQhKOh4vustXrw4+vTxxcmTx/HPP4+zLefi8glsbW1x5swpNGvWQrP+1KmTMDIyQtWqVfMdS3bq1q2HoKBAlClTlkN7esKkioiIPhr16nlh584dWLDgezRt2hx//vknTpw4plNdc+fOhpWVAu7u1WBlpcCtW3/g4cMH+PzzHtnuI5PJMHDgUCxcOA+2trZo0KAR/vrrT2zbthk+Pr21JqmLrXfvvggKOo6RI4fAx6cXHB1LIioqErdv/wl7ewf06tW3wI5NaZhUERHRR6NBg0YYPXoc9uzZhaNHD6N6dQ/Mn78YPXp0znNd1arVwKFDB3Do0AEkJyehVCknjB8/ER075lxXjx49IZfL8dNP27Bv3x7Y29tj8ODhGDBgUI775Ze1tQ3Wr9+MVatWYPnyJYiOjoatrR3c3auhadMW76+A8k0iCIKg7yAKmkqlRkREfK7KyuVS2NpaIDIyHkqluoAjM0xFvQ/Y/qLdfsDw+8DOzgIyWdZf3k5KSsKjRyGwty8BY2OTDxwZ0ccnJSUZr1+Honz5cu+94z1vqUBEREQkAiZVRERERCJgUkVEREQkAiZVRERERCJgUkVEREQkAoNKqvbv3w83N7dM/+bPn6/v0IiIiIhyZJD3qVq3bp3WM44cHR31GA0RERHR+xlkUlW1alU+7JGIiIgKFYMa/iMiIiIqrAzySlX79u0RGRmJUqVKoUePHhgyZAhkMlm+6pTLc5c/pt+lOLu7FRcFRb0P2P6i3X6AfWAoLl36BVu3bsLjxyGIj4+Hg0NxNGnSDEOGDIOlpVWO+16/fg2jRw+DhYUlDh48pjWlJDj4LPz9J2L//qMoVapUQTdDVMHBZ/Hq1St065b98wfTffPNTAQGHoG3dzvMnPmt1rbJkycgNjYWK1euzdPxO3duh0aNGmPSpADR90s/Zxs3bkPlylXyVL+hMKikysHBAWPHjkWNGjUgkUhw5swZ/PjjjwgLC8OMGTN0rlcqlcDW1iJP+ygUfMJ3Ue8Dtr9otx9gH+hbdHQ0qlRxR/fuvWBtbY2QkIdYt24NQkIeYcmSFbmqIz4+Djt3bsfQoSMKONoPIzj4HO7e/TtXSVW6kydPYPDgYXB2Lp3v43///QJYWSnyXc/HyqCSqsaNG6Nx48aa5UaNGsHExASbN2/GiBEjULx4cZ3qVasFxMQk5KqsTCaFQmGGmJhEqFSG98yvD6Go9wHbX7TbDxh+HygUZkXiKpq3dzut5dq168DIyBhz587Gq1ev4ODg8N46ateugz17dqJ3776wsLAsqFBzlJSU9N5nxhWU0qVdkJycjM2bN2Dq1Jn5rs/NrZIIUenHhzgPBpVUZcXb2xsbNmzAnTt3dE6qAOT5oagqldogH6T6IRX1PmD7i3b7gY+rDwS1CkJKpN6OLzG2hUSav2kcAGBtbQ0ASE1NzVX5Pn36YcqUydi9eycGDhySbTlBELBjx1YcPLgfoaH/wcGhOLp390GvXn01Zf755zHWrVuNW7duIjo6GiVLlkSHDp3Rq1cfSKVpSe6LFy/QtWt7TJv2NW7duolz587AwcEB27fvRkpKCtavX42goOMIDw9HqVJOGDRoKNq29dYcIyTkEZYu/RG3b/+F5ORkODo6okOHTvD1HaAZzgMAL69aAIDPPuuAGTNmZdsuIyMj9OjRE4sXL8SgQUNRsmT2w50vX4Zh+fIluHLlMpKSElG5chV88cVEVKr0diguq2G8Awf2YvPmDYiMjEL16jUwZsx49O/fG9OmfY327TtqHWPv3l3Ytm0zYmPjULt2HUyZMh22trZaZSIjI+DvPxFXr16GhYUlunf3wYABg7XK3LhxHStWLMX9+/dgamqGxo2bYOxYP83rI6fzcPPmH1i5cikePHgAQVCjZMlS6N3bF+3adci2b3LL4JMqIiLKn5R/DiLx6mQISa/0FoPE1AFmnvNgXKZznvdVqVRQKpV4/DgEGzasRePGTXM9F8rW1g5dunTFzp074OPTG+bm5lmWW7jwBxw+fBADBgxC1arV8OefN7F8+RKYmJiia9duAIBXr17ik08+Qdu23jA3t8CDB/ewdu1qJCYmYMiQ4Vr1rVy5DA0aNMK3386BWp2WmE+d6o+bN//A4MHDUKZMWVy+/Au+/noarKwUaNCgIQBg0qQvYGdnh6lTZ8DS0hJPnz7Fq1dhAIBBg4YgKioST578g1mzZgMAbGy0E5KsdOzYBZs2bcDmzRsQEDAtyzIxMTEYNmwQzM3NMXHiZFhaWmL37p0YPXoE9uw5mO038s+fD8b3389Bx45d0KJFS9y/fx9Tp/pnWfbChfN4+vRfTJoUgKioKCxevAALFnyP2bPnapX77rvZaNOmLbp0mY/ffruKVauWQ6Gw1pyHu3f/xrhxo1CrVm3873/fIyIiAitWLEFISAjWrt2oNQf73fMQHx+HiRPHoUYND3z77RwYGRnh8ePHiIuLfW8/5obBJ1WBgYGQyWSoUqVwTlojItK3hEvjgdQYvcYgJL1CwqXxOiVVnTu3w6tXLwEAXl4N8M03c/K0f58+/bF//17s27cbvr4DMm1/9uwp9u7dBX//r9C58+cAgHr1PJGUlIT169egc+eukEqlqFvXE3Xreqa1RxBQo4YHkpKSsHfvrkxJVcWKrpg69e1c4OvXf8OFC8FYvHg5PD3rAwA8Pb3w+vVrrFu3Cg0aNERUVCRevHgOP79JaNy4KQCgdu26mjqcnUvDxsYWoaH/wd29eq7bb2Jigj59+mHlyqUYNGgoihfPfO/HXbu2Iy4uFhs2bNUkUHXq1EOPHp2xffsWjB37RZZ1b9q0DnXq1MVXX00HkHZ+lEol1qzJPOdNEAT88MOPMDY2BgD8998LbN68AWq1WnOlL+24dTF2rJ+mvoiICGzcuE5zHjZuXI9ixYphwYLFkMuNAKTdz3L8+NG4dOkXTd8Bmc/DnTt/Iy4uDiNHjkWFChUBQHNOxWBQg/KDBw/GmjVrEBwcjODgYMyYMQObNm1C3759czV2TkREH5+FC5dg7dqNmDJlOv755zEmTRoPlUoFAFCr1VAqlZp/6VeFMnJwcECHDp2wY8c2JCUlZtr+229XAQDNm7fUqqtuXU+Eh79GWFgoACA5ORlr165Et24d0bixJxo1qodVq5bj9evXSEjQnrfbsGEjreWrV69AobBG7dp1tY5Rr54X7t27B5VKBWtrG5QoURIrVy7DsWNH8PJlmCj9BwBdu34OS0tLbNmyMcvtV69eQe3adaFQKDSxSaVS1KxZG3fu/J3lPiqVCvfu3dNKYgCgSZOmWZavWbOWJqECgLJly0GpVCIyMkKrXNOmzbWWmzdvhVevXmr64+bNG2jcuJkmoQIAT8/6sLKyws2bf2jt++55cHJyhoWFJebN+w6nTp1EZKS4Q+IGdaWqbNmy2LdvH0JDQ6FWq1GmTBl89dVX8PX11XdoRESFlnmDxQYz/KeLihVdAQDVqtVAlSpV4evbE8HBZ9GiRSusX78G69ev0ZQdPHhYlt/08/UdgMOHD+LAgX0oVcpJa1tUVBQEQUDbti2yPH5YWBhKliyF5cuX4NChAxg8eBgqVaoMKysrnD9/Dhs3rkNKSrLW0KKdXbFMx4iJiUajRvWyPEZ4+GsUL+6IJUtWYNWq5Zg/fy4SExNRqVJljB8/ATVr1s5dZ2XD1NQMvXr1xfr1azLNT0qP76+//swyPmdn5yzrjIqKhEqlzDQEaWub9VBhxttaAGnzvQAgOTklx/3Tr5y9fv0aJUqURGxsbJbDkXZ2xRATE5NpXUYKhQJLlqzA2rWrMGvWdKhUKtSoURMTJ07WXLnKD4NKqqZNy3qsl4iIdGdcpjOMXDp8FBPVK1SoCLlcjmfPngIAOnf+HI0aNdFst7fPelSjRImS+Oyz9ti+fQvGj5+otU2hsIZEIsHq1Rs0H/QZubh8AgA4c+ZndOnSFf36DdBsu3jxQpbHk0gk7xxDAVtbWyxcuDTL8umJhIvLJ5gzZx6UylTcunULK1cuw6RJfjhy5ES288Fy6/PPe2Dbti3Ytm1zpm0KhTW8vEpj+PBRmbZl1SdA2nwumUyOqCjt19W7V57y6t39IyLSlu3t7d/EqsjyClNERDgUCu3bPbx7HgCgalV3/PjjMiQlJeH3369hyZJF8PefiH37DucrbsDAkioiIioYEqkMElN7fYeRb7dv/wmlUqm52uTg4JDr6SH9+w/C0aNHcOjQfq31deumXZ2Jjo7KNJSVUXJystaQk0qlws8/B+Xq2HXremLbts2Qy+WaK285kcuNUKtWbfTrNwBffumH169fwcXlExgZyZGSkpyrY77LwsICPXv2xubNG+Hq6qrVlrp16yEoKBBlypSFmVnu7s8mk8ng5uaG8+fPwcent2b9+fPndIovXXDwWTRr9vaq4dmzp+Dg4KCZC1a9ugfOnz+LceP8IJenpTFXr15BbGwsatTwyPVxTE1N0aBBIzx79gyLFv2A5ORkmJiY5Ct2JlVERGSQ/P0nonLlKqhQoSJMTEzw4MEDbN++BRUqVMw07yY3SpVyQtu23prbEqRzcfkEn3/eA7NmTUefPv1QtWo1KJVKPH36BNevX8O8eQsBAPXqeeHw4QMoW7YcbGxssG/fnlzf2sHT0wuNGjWBn98Y9O3bHxUqVERiYiIePw7B06dPMXXqDDx4cB9LlixCq1Zt4OzsjLi4OGzevBElS5aCk1PaEFyZMmVx9OhhnDx5AqVLu8Da2iZPd4Xv0aMnduzYij//vKU1pNi7d18EBR3HyJFD4OPTC46OJREVFYnbt/+Evb2D1q0lMhowYAgmT/bDnDnfomXLVrh3756mfzNOPs+La9d+w9Kli1Cvnhd+/fUKjh8/hkmTAjT1DRw4GEOHDsSkSePRvXtPRESEY8WKpahSxR0NGjTKse6LFy/g8OGDaNasORwdSyI8/DX27NmJ6tVr5DuhAphUERGRgapa1R2nTp3Eli2bNPcT6tSpC/r08c12SOp9BgwYhKCgQM1E93QTJ07GJ598goMH92PDhrUwMzPHJ598ghYtWmnKTJgwGfPmzcGCBfNgamqKdu06oGnT5vjuu2/fPUyWvvvuB2zZshH79u1BaOh/sLS0RLly5TX3cipWzB7FihXDli0b8erVS1hYWMLDoya+/nq25jYBHTt2xt9/38aCBfMQHR313vtUvcvS0grdu/fExo3rtNZbW9tg/frNWLVqBZYvX4Lo6GjY2trB3b0amjbNeq4ZkDYpffLkr7B58wYEBQWiShV3fPnlVxg/fhQsLXW72WpAwFQcPLgf+/btgbm5BYYNG6V1B/lKlapg8eIVWLlyKaZM+VJzn6px4ya895F2zs6lIZVKsWrVCkRGRsDa2hr16nlh1KixOsX6LokgCIIoNRkwlUqNiIj4XJWVy6WwtbVAZGT8R3PTv7wq6n3A9hft9gOG3wd2dhbZ3lE9KSkJjx6FwN6+BIyN8/+XN1FeHT58EHPmfFMon62YlZSUZLx+HYry5cu9947svFJFREREOomOjsb69WtQp05dmJub4++//8amTevRpEmzjyKhyismVURERKQTuVyO58+f4uTJ44iNjYOtrQ28vT/D6NHj9R2aXjCpIiIiIp1YWFhgwYIl+g7DYBjUHdWJiIiICismVUREREQiYFJFREREJAImVUREREQiYFJFREREJAImVUREREQiYFJFREREJAImVUREZPD69vWBl1ct/PHH7/oORWcvXryAl1ctnDlzSt+hUAHJc1KVmJiIMWPG4PDhwwURDxERkZaQkEd4+PABACAo6ISeo9Gdvb091q3bhNq16+o7FCogeU6qzMzMcOnSJSQlJRVEPERERFqCggIhlUpRu3YdnDlzCkplaoEfUxAEpKSkiFqnsbEx3N2rw9raWtR6yXDoNPxXu3Zt3LhxQ+xYiIiogKgENV4nx+vtn0pQ6xS3IAg4eTIItWvXRa9efREdHYXLly8BALp0aY/58+dm2mfJkkXo0OFTqNVpx0xJScHKlUvRufNnaNzYEz4+XREUdFxrn2++mYnevbvj0qVf0LevDxo39sQvv5xHYmIi5s+fix49uqBp0wbo3Lkdvv/+f4iLi9XaPzU1FQsWzEObNs3QqlUTzJ07G0FBx+HlVQsvXrwAkPXwX+fO7TB//lzs3bsLnTt/hpYtm2Dy5AmIjIzUqj8k5BFGjBiMJk280K1bJ5w4EYjJkydg5MihOvUrFQydnv03Y8YMDB48GIsWLUKvXr1QokQJseMiIiKRHHp+GwG3juFVcrzeYnAwscDc6u3Qyalqnva7desm/vvvBQYPHgovr/qwtrbByZMn0LhxU7Ru3RZHjx6Gn9+XkMlkANKSsFOnTqJVqzaQStOuG0yd6o+bN//A4MHDUKZMWVy+/Au+/noarKwUaNCgoeZYr1+/wsKF8zBgwBCUKFECjo4lkJSUBJVKjREjRsPGxhZhYaHYtGk9Jk+eiBUr1mj2Xb58CQ4e3IehQ0fC1dUVZ86cxooVS3PVxgsXzuPp038xaVIAoqKisHjxAixY8D1mz05LGJOSkjB+/ChYWlrh669nAwDWr1+D2NhYODk556k/qWDplFR17NgRKpUKa9aswZo1ayCTyWBsbKxVRiKR4Pr166IESUREuvO7cRgxSv1O2XiVHA+/G4fznFSdPHkcJiYmaNasBeRyI7Ro0RLHjx9DQkICWrduiy1bNuLatd/g6ekFAPjjj9/x8mUY2rT5FABw/fpvuHAhGIsXL4enZ30AgKenF16/fo1161ZpJVUxMTFYuHAp3N2racXg7/+V5melUolSpZwwfPgg/PvvE7i4fILo6GgcOLAXAwcOQb9+AwAAXl4NMGbMCISFhb63jYIg4IcfftR8jv733wts3rwBarUaUqkUR48eRkREBFav3ohSpUoBACpVqoLu3TsxqTIwOiVVbdu2hUQiETsWIiIiDaVSiTNnTqF+/YawtLQCALRp8ykOHNiH4OAz8PZuj7Jly+HUqSBNUvXzzydRurQLKleuAgC4evUKFApr1K5dF0qlUlN3vXpe+P77OVCpVJqrXNbWNpkSKgA4fvwofvppO54+/ReJiYma9elJ1aNHD5GcnIzGjZtq7dekSTNcu/bre9tZs2YtrQsTZcuWg1KpRGRkBIoVs8edO7dRvnwFTUIFAKVKlUKFChXfWzd9WDolVXPnZh7DJiIiw7SoZkeDGf7Li19/vYLIyEg0atQEsbFpc5jKl68Ie3t7BAWdgLd3e7Ru/Sl++mkrJk/+ChKJBGfPnkLXrt00dURFRSEmJhqNGtXL8hjh4a9RvLgjAMDOzi7T9nPnzmDWrBno3LkrRowYDYXCGuHhr+HvP1EzkT08/DUAwMbGVmtfW1vbTPVlxcrKSmvZyMgIAJCc/Lb+d+tOq98OycnJuToGfRg6JVVERFR4dHKqivalKiMyJfH9hQuIrbEZZJK8fTcqfTL57NlfY/bsr7W2RUZGISIiAq1bt8GaNStw+fIlGBsbITIyEq1bf6opp1AoYGtri4ULs57fZGv7NpHKagTmzJlTcHV1Q0DANM2633/XntpSrJg9ACAqKhIODg4ZYtSebK6rYsXs8eDB/UzrIyMjYG5uIcoxSBw6J1VxcXHYtGkTzp07p/lmQ6lSpdCsWTMMGDAAlpaWogVJRET5I5NIYW9SeD6Ak5ISceFCMJo2bY4ePXppbYuICMf06VNw6tRJ9OjRE1WqVMXPP5+AkZExXF3dUKZMWU3ZunU9sW3bZsjlclSs6JrnOJKTkyGXG2mtCwoK1FouX748TExMcP78Oa1jnD9/Ns/Hy0rlylVx/PgxvHjxHKVKOQFI+ybhw4cPUL26hyjHIHHolFSFhYWhT58+ePbsGcqVK4datWoBAB4/foxly5bh0KFD2L59O4oXLy5qsEREVDScPx+MhIQE9OjRE7Vr18m0fevWzTh58jh69OiJ1q3bYs2alZDJZBgwYLBWOU9PLzRq1AR+fmPQt29/VKhQEYmJiXj8OARPnz7F1Kkzcoyjbl1PzJ8/Fxs2rIW7e3VcuvQLrl37TauMtbUNunTphk2b1sPYOC2xO336FP79918AgFSavznI7dt3xObN6zFx4ngMHToCALBu3WrY2RXLd90kLp3uUzV//ny8fv0aq1evRmBgIJYtW4Zly5bh2LFjWL16NV6/fo0FCxbkK7D4+Hg0adIEbm5u+PPPP/NVFxERFS5BQcdRokQJ1KqVOaECgHbt2uOvv/7Es2dP0apVWyQnJyM+Ph6tW7fNVPa7735Aly7dsG/fHvj5jcWcOd/g6tXLmgsCOenS5XP07u2LPXt2IiBgEl6+DMOsWf/LVG706HHo1KkrtmzZiK++8odSqdR8EzC/IzempqZYvHgFFAoFZs6cimXLFqN3774oXdpFM4GfDINEEAQhrzt5eXnBx8cHfn5+WW5fuHAhdu/ejStXrugc2A8//ICDBw/i9evX2Lt3L6pVy/yNjNxSqdSIiMjdBE25XApbWwtERsZDqdTtZnWFXVHvA7a/aLcfMPw+sLOzgEyW9d/ESUlJePQoBPb2JWBsbPKBI6OMvv56Gm7e/AMHDhwVve7o6Gh8/nkH9OzZB0OGDBe9fnorJSUZr1+Honz5cjA1Nc2xrE7Df4mJiShWrFi22+3t7bW+dppXjx49wo4dO+Dv74+ZM2fqXA8REdGH8Pvv13Hr1h+oVKky1GoBFy+eR1DQcYwfP0GU+rds2QQ7OzuULFkK4eGvsX37FqjVanTo0EmU+kkcOiVV5cuXx7Fjx9CzZ89MN/1MTU3FsWPHUL58eZ2Dmj17Nnr27ImyZcu+vzAREZGemZub4eLFC9i6dTOSk5NQqpQTxo+fgJ49+4hSv1QqwaZN6/Dq1SvIZDJUqeKOZctWw9GRTzQxJDolVUOHDoWfnx+6d++O3r17o0yZMgDSJqrv3LkT9+7dw6JFi3QK6MSJE7h//z6WLl2K27dv61RHVuTy3E0fS7+knt2l9aKgqPcB21+02w+wDyjvKlWqgrVrNxVY/X379kffvv0LrH4Sh05Jlbe3NxITE7FgwQLMnDlTc28PQRBQrFgxzJkzB59++ul7asksMTERc+fOhZ+fn6i3ZJBKJbC1zdtXiRUKM9GOX1gV9T5g+4t2+wH2ARHlTZ6TKkEQEB8fj3bt2qFjx47466+/tO5T5e7uDrlct9tfrVy5EsWKFcPnn3+u0/7ZUasFxMQk5KqsTCaFQmGGmJhEqFSGN0H1QyjqfcD2F+32A4bfBwqFWS6uouX5O0hElKXcv5fynP2kpqaiXr168PPzw9ChQ+Hh4QEPD4+8VpPJ8+fPsWHDBixfvlzzOIKEhATNf+Pj42FhofuN6/L6DR6VSm2Q3/r5kIp6H7D9Rbv9QOHsAyMjI0gkaTetNDbO+ZtKRPR+ycnJkEjePj4oJ3lOqoyNjWFvb59pgnp+PXv2DKmpqRg2bFimbf369UONGjWwe/duUY9JRPSxkclksLGxQWRkFADAxMQEAG8QSZR3ApKTkxEbGwVbWxvNg7dzotM4XZcuXXDo0CH06tVLtOSqcuXK2LJli9a6O3fu4LvvvsOsWbPydZ8qIqKipGTJkgDSHib85sI/EelAIgFsbW0076n30SmpcnNzw+nTp9G+fXt06dIFTk5OWd4Qq02bNrmuU6FQwNPTM8ttVatWRdWqVXUJlYioyJFIJChVqhQcHR2Rmpqq73CICi0jI6NcXaFKp1NSNWHC25uZLV68OMsyEokEd+7c0aV6IiISgUwmy9MHAhHlj05J1bvDdAXF09MT9+7d+yDHIiIiIsqPPCdVycnJuHv3LipXroy6desWRExEREREhU6ebxdsYmKC+fPn4/HjxwURDxEREVGhpNMzGCpWrIjnz5+LHQsRERFRoaVTUuXn54edO3fi0qVLYsdDREREVCjpNFF927ZtsLGxweDBg+Hs7AxnZ+c3N5h7SyKRYOXKlaIESURERGTodEqq7t+/DyDtBnMqlQpPnjzJVCb9IctERERERYFOSdWZM2fEjoOIiIioUNNpThURERERadPpShUAqFQqnDhxAlevXkV4eDjGjRsHNzc3xMbG4vLly6hVqxbs7e3FjJWIiIjIYOmUVMXExGDIkCG4desWzM3NkZiYiL59+wIAzM3NMXv2bHTu3FnrcTZEREREHzOdhv/mz5+PBw8eYP369Th16hQEQdBsk8lkaNu2LYKDg0ULkoiIiMjQ6ZRUnT59Gr6+vmjYsGGW3/IrU6YMbw5KRERERYpOSVVsbCycnZ2z3a5UKqFSqXQOioiIiKiw0SmpcnFxwe3bt7PdfvHiRZQvX17noIiIiIgKG52Sqm7dumHfvn0IDAzUzKeSSCRISUnBokWLcOHCBfj4+IgaKBEREZEh0+nbf/3798fDhw8xYcIEKBQKAMCkSZMQFRUFpVIJHx8fdO/eXdRAiYiIiAyZTkmVRCLR3DYhKCgIT548gVqthouLC7y9vVG3bl2x4yQiIiIyaDrf/BMA6tSpgzp16ogVCxEREVGhxcfUEBEREYmASRURERGRCJhUEREREYmASRURERGRCJhUEREREYmASRURERGRCHJ1S4UpU6bkuWKJRII5c+bkeT8iIiKiwihXSdXVq1czrUtKSkJERAQAwNraGgAQHR0NALCzs4OZmVmegwkODsbatWvx8OFDxMXFwdHREa1atcKYMWNgZWWV5/qIiIiIPpRcJVVnzpzRWn748CEGDRqE4cOHo3///rCzswMAREREYPPmzTh48CDWrFmT52CioqJQvXp1+Pr6wsbGBg8ePMDSpUvx4MEDbNiwIc/1EREREX0oOt1R/dtvv0WTJk3g5+entd7Ozg5+fn4IDw/Ht99+i02bNuWp3k6dOmkte3p6wtjYGNOnT0dYWBgcHR11CZeIiIiowOk0Uf3mzZuoUqVKttsrV66Mmzdv6hxURjY2NgCA1NRUUeojIiIiKgg6XamytrbG+fPn0bt37yy3nz9/Pl9zoFQqFZRKJR4+fIjly5ejRYsWcHZ21rk+AJDLc5c/ymRSrf8WRUW9D9j+ot1+gH1ARLrRKany8fHBkiVLMHLkSPj6+sLFxQUA8OTJE2zduhXnz5/H2LFjdQ6qefPmCAsLAwA0btwYCxYs0LkuAJBKJbC1tcjTPgpF3ifaf2yKeh+w/UW7/QD7gIjyRiIIgqDLjj/++CPWr18PpVKptV4mk2Hw4MGZ5lvlxd27d5GYmIiHDx9i5cqVcHZ2xsaNGyGTyXSqT6VSIyYmMVdlZTIpFAozxMQkQqVS63S8wq6o9wHbX7TbDxh+HygUZryKRmSAdE6qgLRv+126dAkvXrwAADg5OaF+/fqabwOK4e7du+jUqRMWL16MTz/9VKc6VCo1IiLic1VWLpfC1tYCkZHxUCoN75fph1DU+4DtL9rtBwy/D+zsLJhUERkgnYb/0tnZ2aF9+/ZixZIlNzc3GBkZ4d9//y3Q4xARERHlh85JlUqlwokTJ3D16lWEh4dj3LhxcHNzQ2xsLC5fvoxatWrB3t4+3wHevHkTqamp+Z6oTkRERFSQdEqqYmJiMGTIENy6dQvm5uZITExE3759AQDm5uaYPXs2OnfujAkTJuSp3jFjxsDd3R1ubm4wNTXF3bt3sX79eri5uaFVq1a6hEpERET0QeiUVM2fPx8PHjzA+vXrUblyZTRo0ECzTSaToW3btggODs5zUlW9enUEBgZizZo1EAQBTk5O6N69OwYPHgxjY2NdQiUiIiL6IHRKqk6fPg1fX180bNgQkZGRmbaXKVMGBw4cyHO9w4YNw7Bhw3QJiYiIiEivdPr6SGxsbI5znJRKJVQqlc5BERERERU2OiVVLi4uuH37drbbL168iPLly+scFBEREVFho1NS1a1bN+zbtw+BgYFIv82VRCJBSkoKFi1ahAsXLsDHx0fUQImIiIgMmU5zqvr374+HDx9iwoQJUCgUAIBJkyYhKioKSqUSPj4+6N69u6iBEhERERkynZIqiUSiuW1CUFAQnjx5ArVaDRcXF3h7e6Nu3bpix0lERERk0PKcVCUmJuLLL79EmzZt0LFjR9SpU6cg4iIiIiIqVPI8p8rMzAyXLl1CUlJSQcRDREREVCjpNFG9du3auHHjhtixEBERERVaOiVVM2bMwPXr17Fo0SKEhoaKHRMRERFRoSMR0u+JkAc1a9aESqVCamoqgLRH07z7GBmJRILr16+LE2U+qVRqRETE56qsXC6Fra0FIiPjoVSqCzgyw1TU+4DtL9rtBwy/D+zsLCCT6fQ3MREVIJ2+/de2bVtIJBKxYyEiIiIqtHRKqubOnSt2HERERESFGq8fExEREYlApytV6UJDQ/H3338jNjYWWU3N6ty5c36qJyIiIio0dEqqkpOT4e/vj5MnT0KtVkMikWg9AzAdkyoiIiIqKnQa/lu4cCF+/vlnfPHFF9i6dSsEQcDcuXOxYcMGNGnSBJUqVcKhQ4fEjpWIiIjIYOmUVAUFBaFr164YNmwYKlSoAABwdHREgwYNsHr1alhZWWH79u2iBkpERERkyHRKqsLDw1G9enUAgKmpKYC0ZwKma9u2LX7++WcRwiMiIiIqHHRKquzt7REZGQkg7VmA1tbWePz4sWZ7XFwckpOTxYmQiIiIqBDQaaJ69erV8fvvv2uWmzdvjvXr18PBwQFqtRqbNm2Ch4eHWDESERERGTydkipfX1+cOHECKSkpMDY2xvjx43Hjxg1MnjwZAODi4oKpU6eKGigRERGRIdMpqapTpw7q1KmjWS5ZsiSOHz+O+/fvQyqVoly5cpDL83ULLCIiIqJCRbTMRyqVolKlSmJVR0RERFSo6JRU/fbbb7kqV7duXV2qJyIiIip0dJ5TlfHO6dm5c+dOnuo9fvw4Dh8+jNu3byMmJgaffPIJfH198fnnn+fqeERERET6olNStWXLlkzrVCoVnj9/jt27d0OtVmPixIl5rnfTpk1wcnJCQEAAbG1tcenSJUyfPh2hoaEYM2aMLqESERERfRA6JVX16tXLdlvXrl3Ru3dv/Prrr6hfv36e6l25ciXs7Ow0y/Xr10dUVBQ2btyIUaNGQSrV6bZaRERERAVO9CxFKpWiXbt22LNnT573zZhQpatcuTLi4uKQkJAgRnhEREREBaJALv1ER0cjNjZWlLquX78OR0dHWFpailIfERERUUHQafjvxYsXWa6PiYnBtWvXsH79eq37WOnq2rVrCAwMhL+/f77rkstzlz/KZFKt/xZFRb0P2P6i3X6AfUBEupEIgiDkdadKlSpl+208QRDg4eGB+fPnw9nZWefAQkND0b17d5QvXx4bNmzI13wqQRD47UEiIiIqUDolVfv378+UpEgkEigUCri4uKBChQr5CiomJgZ9+vQBAOzYsQNWVlb5qk+lUiMmJjFXZWUyKRQKM8TEJEKlUufruIVVUe8Dtr9otx8w/D5QKMx4FY3IAOk0/Ne1a1ex49BISkrC8OHDERsbi127duU7oUqnVObtF6NKpc7zPh+bot4HbH/Rbj/APiCivDGoB/QplUp88cUXCAkJwfbt2+Ho6KjvkIiIiIhyRaekql+/fnneRyKRYPPmzTmWmTVrFs6ePYuAgADExcXhjz/+0GyrUqUKjI2N83xcIiIiog9Bp6RKEASEhobi6dOnsLKyQunSpQEAz549Q0xMDFxcXDJdZcrN1K2LFy8CAObOnZtp2+nTp/M18Z2IiIioIOmUVI0fPx6jRo3Ct99+iy5dukAuT6tGqVRi//79mD9/Pr777jvUrl07T/WeOXNGl3CIiIiI9E6nr4/MmzcPXbt2Rffu3TUJFQDI5XL06NEDXbt2zfJqExEREdHHSqek6t69e5ohv6w4Ozvj/v37OgdFREREVNjolFQVL14cgYGBUCqVmbYplUoEBgaiePHi+Q6OiIiIqLDQaU7VkCFDMHPmTPTo0QO9evWCi4sLAODJkyfYuXMn7ty5g5kzZ4oaKBEREZEh0ymp8vHxgVQqxY8//ojp06dr7q4uCALs7Owwa9Ys9OjRQ9RAiYiIiAyZzjf/7N69O7p06YK//vpL84DlUqVKwd3dXWvyOhEREVFRkK/sRy6Xw8PDAx4eHiKFQ0RERFQ46TRR/c6dOzh69KjWugsXLqBPnz7o3r37e++cTkRERPSx0Smp+uGHHxAYGKhZfvr0KcaMGYNnz54BSLsj+q5du8SJkIiIiKgQ0Cmpunv3rtbd0g8dOgSpVIoDBw5gz549aNu2LXbu3ClakERERESGTqekKjY2FjY2Nprl4OBgNGzYEHZ2dgCAhg0b4smTJ6IESERERFQY6JRUOTg44NGjRwCAly9f4vbt22jYsKFme3x8PKRSnaomIiIiKpR0+vZfy5YtsW3bNqSkpODmzZswNjZG69atNdvf9xgbIiIioo+NTknVF198gYiICBw6dAhWVlb47rvvYG9vDwCIi4vDiRMn0KdPH1EDJSIiIjJkOiVVFhYWWLBgQZbbzM3Ncf78eZiamuYrMCIiIqLCRPRbn0ulUlhZWYldLREREZFB42xyIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhGIfkf1/Hjy5AnWr1+Pmzdv4sGDByhXrhyOHj2q77CIiIiI3sugkqoHDx4gODgYNWrUgFqthiAI+g6JiIiIKFcMavivRYsWCA4OxpIlS1C1alV9h0NERESUawaVVEmlBhUOERERUa4Z1PBfQREEAampqVluk0gkkMvfdkNqagpSUoyQmpoCpVJ4pywglxtlKJt1nXktCwBGRrqVVSpTkdMoqe5llVn2QXZlcxqqlcvlkEgkAACVSgm1uiDKqqBWq0UpK5W+bdv7yspkMs0fAwVVVq1WQ6VS5RCvFDKZTLSygiBBSorRm22SPNcrCAKUSqVIZSWQyeSil838vtd+z6X3QWpqClQq5FhWu94P8zuCUyOIDFORSKpiY2Owdu3SLLeVKVMWHTt+rllesWIllMqsf7k5OTnj8897apY3bFiHpKTELMsWL+6Inj19Nctbt25GbGxMlmXt7Iqhb9+BmuWdO3cgIiI8y7JWVgoMHDhMs7x37268fBmWZVlTUzMMGzZas3zo0AE8f/4sy7JyuRyjRn0BAJDJpNi9ezcePHiQZVkAGDdukubnkydP4OHD+9mWHTlyHORyYwDA2bOncefO7WzLDhkyCubm5gCACxfO488//8i27IABQ6FQWAMArlw5j99/v5Zt2T59BqBYMXsAwLVrl/Hrr5ezLdurly+KFbOCTCbFX3/dwMWL57Mt27VrDzg7uwAAbt++ieDg09mW7dChC8qWLQ8AuH//b5w6dSLbst7eHVCxohuAtLmGx48fybZsq1afokoVdwDA48ePceTIgWzLNm3aEjVq1AQAPHv2DPv37862bOPGTVGzZl0AQFhYGHbt2p5t2Xr16sPLqyEAIDz8NbZv35Rt2Vq16qBRo2YAgJiYaGzatDbbstWqeaB581YAgISEBKxbtyLbspUrV0Xr1t4A0v44yu49DwAVKrjis886apZXrMi+7Lu/I9asWZltwvahfkfEx8fB2to625iJSD+KRFKVEyMjOWxtLTTLby5mZEkul2mVlUqzL5yXsjKZVKusTJb9MKhUKtEqK5fLRCkrkWiXfZ+MZY2Nc34Z2dhYwNjYOJdlzWFhkVa3iUnOZa2tzWFjk17WKMeyCoWZJmYzM+Mcy1pYmGj2eV9ZK6u39Zqb51zW0tJUUzb9GDnFoEvZV69Mcyxrbm6sKRsdbZZjWRMTI03ZhIScy5qZva03NTU+1/VKJDlfnTExefv+NM65e2Fs/LZsSkrOr4eMZd8n8+8Icd73+fkdER0d/d64iejDkwgGeh05ICAAf/31lyi3VFAqVYiIyPovQIlEqnVpX61WwsrKDLGxiVCp1O+UlbxzuT4l22NmLpsKILuulmQx/Je7smlDetmfQiMj4zyXlcmkMDc3QnR0fKY+yLpeJQQhp6E3I80HUUGVTRtOy354Ki9ljY2NYWNjgZiYRKSkpOZYViaTvzOkJ37ZtKG3nIa9ZO8M6eWvrEwmhZWVGeLjU6A9/Je7etOG3rJPlvJSNuP7s6DKApnfy+l9EBubCLX63eG/vLzvC+Z3hEJhqulDIjIcReJKlUQigUSSfVOVyrcf3HK5HMbGxpBKU7Oc85KxbE51Zi6b8y9AXcsCshyvrulaVi6XQyqVZzvvR7teKSSS7K+uqVQC3n5YFFTZnM9xXsqm550qlRqCkHNZtRoZ+qigyub8WhOE3L8uc1NWKpXC2NgY8fGpotarS1lAP2XT+yD998CHiSEv7/sc3shEpDf8uh0RERGRCAzqSlViYiKCg4MBAM+fP0dcXBxOnEibzFuvXj3Y2dnpMzwiIiKibBlUUhUeHo7x48drrUtf3rJlCzw9PfURFhEREdF7GVRS5ezsjHv37uk7DCIiIqI845wqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEwqSIiIiISAZMqIiIiIhEYXFL16NEjDBw4EB4eHmjYsCHmzZuHlJQUfYdFRERElCO5vgPIKDo6Gv3790eZMmWwdOlShIWFYe7cuUhKSsKMGTP0HR4RERFRtgwqqdq5cyfi4+OxbNky2NjYAABUKhVmzZqF4cOHw9HR8YPEkaRMRapaBQkkH+R4REWNIAgAhAxr0t5rEknBv+fSjo23x9eKJe2/gloKQZUKQa1M2wzJB4mNiAo3g0qqzp8/j/r162sSKgDw9vbGzJkzcfHiRXTt2rXAY/jyzBxsjk6EWiKFTFDDBCoYQwUTQQVTqGAiKNOWoYKpoIIxlDARVDCCGjIIkAKQQoAEePNz2rJUkvaxIRUAqUTQbJNASNsOAVJBeLOvoL3+zTYJ1FppniTDh5IEAiBor0v/gJBoymTcN+3/044l0dQhQAJpWkVvPmwAQP2mqrQPH0nG9RmOo30UaVoVEolmnSCRZohB+mZPCdI/0jQ/S/CmB99Zn34EQYAU6rTYBfWbPlOnrUvvQ0H9toxWbNI34UgASfqZevNPktbrkLzZJghaNQiStDjUmtgkWv8AARDUb46khiTTz+n9p35zvtJblh5BxmiELNa9Xca7iUGGn9PLQWuLBILkzSvuzTlJW5a+KSl9p/0SqIS046RF/fZ4b1a/XS+kn+I3ZynD61C7nYLmaHlLTzKUlkg0ayRCxvcKNP399j309v2V/joC8Oacpi2rJW9fZ2pNbW/LakeheQW+iUPypu1v32ESScYz9Saa9HWSN/2sqU0Ce2NTtK7nD6PinnnqESIyTAaVVIWEhODzzz/XWqdQKODg4ICQkJB81S2Xv3/6mFqtxqGoKKilZgAAlUSKBEiRAKO8fgoQEeVKn6s/YXGX+voOg4hEYFBJVUxMDBQKRab11tbWiI6O1rleqVQCW1uL95YTBAENYk0RZhaKJIkMyZAhRSJDikSKFM3PMqRApvkLl4goPx7LK+bq9xMRGT6DSqoKilotICYmIVdlZ3X+CoGPIhATn6wZ4kgfakkf9lALAtQQoBSUSBWUUEEFpVoNAWnr04dF1G+GfNRvltPWCZqhFJWQvg8gpP8s4G05QXgzwKaGWgDUmuG2DCNYkACSdwcr0oai0vM+SfrQlGYfZHnlTVNeAsikUihVqgx9gLd98KZPMg5ACcKbUSOt2NJ/zjBkk+EYWvGmz1vJsFYQsp7Vpj0Q9aZvhYxLGbZliDNzDVktpZHJZBDUakCQZGhX2rBPejskb/4nk7yN/O3QmJChfuFtP6YFpelHSYZY3zb23ZjfDJlJBM25eDtkKmiWkeHY7y6/25OSHH6SSCSQSiUQ1OlDdW/nO70dzE3vB4n2axFv5yy9jeGdZa2+yWpYOgsZNqSf0/T/qtPfXW86R/3OkKVaSGv/23OWXqU0w7mUap1nqUQCqUwKlUqddpwMJ+TtK007Jk2okqz6VLvf0t8fxc0s8XVtT0RGxmfX8iwpFGaQyQzuy9tERZ5BJVUKhQKxsbGZ1kdHR8Pa2jpfdSuV6vcXAvCJtSm+alkRkZHxud7nYyOXS2Fra1Fk+4DtL9rtBz58HxTVfib62BhUUlWuXLlMc6diY2Px6tUrlCtXTud6pVIJ7OzydnldoTDT+Xgfi6LeB2x/0W4/YLh9IJVy+gGRITKopKpJkyZYtWqV1tyqEydOQCqVomHDhjrXK5FIIJPl7ZcQL62zD9j+ot1+gH1ARHkjETJOBtCz6OhotGvXDmXLlsXw4cM1N//s0KEDb/5JREREBs2gkiog7TE13377LW7cuAELCwt06tQJfn5+MDY21ndoRERERNkyuKSKiIiIqDDihAEiIiIiETCpIiIiIhIBkyoiIiIiETCpIiIiIhIBkyoiIiIiETCpIiIiIhIBkyoiIiIiETCpIiIiIhIBkyoiIiIiETCpIiIiIhIBk6oMHj16hIEDB8LDwwMNGzbEvHnzkJKSou+wPoj9+/fDzc0t07/58+frO7QC8eTJE8yYMQOdOnVClSpV0L59+yzL7dmzB23btkW1atXQsWNHnD179gNHWjBy035fX98sXxOPHj3SQ8TiOn78OEaOHIkmTZrAw8MDnTp1wt69e/HuU7s+1vNPRAVDru8ADEV0dDT69++PMmXKYOnSpQgLC8PcuXORlJSEGTNm6Du8D2bdunWwsrLSLDs6OuoxmoLz4MEDBAcHo0aNGlCr1Zk+TAHg2LFjmD59OkaMGAEvLy8EBgZizJgx2L59Ozw8PD580CLKTfsBoFatWvD399da5+zs/CFCLFCbNm2Ck5MTAgICYGtri0uXLmH69OkIDQ3FmDFjAHzc55+ICgYfqPzG6tWrsWrVKpw9exY2NjYAgF27dmHWrFk4e/bsR5tcpNu/fz+mTJmCy5cvw87OTt/hFDi1Wg2pNO1CbUBAAP766y8cPXpUq0zbtm3h7u6OBQsWaNb17NkTVlZWWLt27QeNV2y5ab+vry/Mzc2xevVqfYRYoCIiIjK9zqdPn47AwED89ttvkEqlH/X5J6KCweG/N86fP4/69etrEioA8Pb2hlqtxsWLF/UXGBWI9IQiO0+fPsU///wDb29vrfWfffYZLl++XOiHhd/X/o9dVn84VK5cGXFxcUhISPjozz8RFYyi/Zs1g5CQEJQrV05rnUKhgIODA0JCQvQU1YfXvn17VK5cGS1btsTq1auhUqn0HZJepJ/zsmXLaq0vX748UlNT8fTpU32E9cH9+uuv8PDwQLVq1dC3b1/89ttv+g6pwFy/fh2Ojo6wtLTk+ScinXBO1RsxMTFQKBSZ1ltbWyM6OloPEX1YDg4OGDt2LGrUqAGJRIIzZ87gxx9/RFhYWJGaU5Yu/Zy/+5pIXy4Kr4m6deuiU6dOKFOmDF6+fIn169dj4MCB2Lp1K2rWrKnv8ER17do1BAYGauaP8fwTkS6YVBEAoHHjxmjcuLFmuVGjRjAxMcHmzZsxYsQIFC9eXI/RkT6MGzdOa7lZs2Zo3749VqxY8VHNKQoNDYWfnx88PT3Rr18/fYdDRIUYh//eUCgUiI2NzbQ+Ojoa1tbWeohI/7y9vaFSqXDnzh19h/LBpZ/zd18TMTExWtuLEnNzczRt2hS3b9/WdyiiiYmJwdChQ2FjY4OlS5dq5prx/BORLphUvVGuXLlMc6diY2Px6tWrTHOt6OOXfs7ffU2EhITAyMgIpUuX1kdYJKKkpCQMHz4csbGxmW4lwvNPRLpgUvVGkyZNcOnSJc1fogBw4sQJSKVSNGzYUI+R6U9gYCBkMhmqVKmi71A+uNKlS6NMmTI4ceKE1vrAwEDUr18fxsbGeopMfxISEnDu3DlUq1ZN36Hkm1KpxBdffIGQkBCsW7cu0y1TeP6JSBecU/VGz549sXXrVowePRrDhw9HWFgY5s2bh549e37096gCgMGDB8PT0xNubm4AgNOnT2P37t3o168fHBwc9Byd+BITExEcHAwAeP78OeLi4jQfoPXq1YOdnR3Gjh2LSZMmwcXFBZ6enggMDMStW7ewbds2fYYuive1Pz3ZaN26NZycnPDy5Uts3LgRr169wuLFi/UZuijS7z8XEBCAuLg4/PHHH5ptVapUgbGx8Ud9/omoYPDmnxk8evQI3377LW7cuAELCwt06tQJfn5+ReKv0tmzZ+PChQsIDQ2FWq1GmTJl0L17d/j6+kIikeg7PNE9e/YMLVu2zHLbli1b4OnpCSDtMSVr167FixcvULZsWUyYMAHNmzf/kKEWiPe1v0SJEvjmm29w7949REVFwczMDDVr1sSYMWNQvXr1Dxyt+Fq0aIHnz59nue306dOau8Z/rOefiAoGkyoiIiIiEXBOFREREZEImFQRERERiYBJFREREZEImFQRERERiYBJFREREZEImFQRERERiYBJFREREZEImFQRERERiYBJVRG3f/9+uLm54dmzZ/oOhQoxX19f+Pr66jsMIiK9YlJFH40jR45g06ZNBX6chw8fYunSpUxEiYhIC5OqIq5Tp064desWnJyc9B1Kvh09ehRbtmwp8OM8fPgQy5Yty/bZcUREVDTJ9R0A6ZdMJoNMJtN3GERERIUer1QVcVnNqWrRogWGDx+Oa9euoVu3bqhWrRpatmyJgwcP5rresLAwfPXVV2jUqBHc3d3RokULzJw5EykpKZoyT58+xbhx41CvXj3UqFEDPXr0wLlz57TquXr1Ktzc3BAYGIiVK1eiSZMmqFatGvr3748nT55oyvn6+uLcuXN4/vw53Nzc4ObmhhYtWmi2p6SkYMmSJWjdujXc3d3RtGlTzJs3Tysef39/VKtWDY8ePdKKYfDgwahbty7CwsKwf/9+jB8/HgDQr18/zbGuXr2aY388evRI09Zq1aqha9euOH36tGZ7eHg4vLy84Ovri4zPOH/y5Ak8PDzwxRdfaNZdu3YN48aNQ7NmzTRtmTNnDpKSkrSOGRAQgJo1a+LFixcYPnw4atasicaNG2P79u0AgHv37qFfv37w8PBA8+bNceTIEa39018bv/32G2bMmAFPT0/UqlULkydPRnR0dI7tzW2fA8DFixfRq1cv1KlTBzVr1kTbtm2xcOHC99ZPRGRoeKWKsvTkyROMHz8e3bp1Q5cuXbBv3z4EBASgatWqqFixYo77hoWFoVu3boiNjUWPHj1Qrlw5hIWFISgoCElJSTA2Nsbr16/Rs2dPJCYmwtfXF7a2tjhw4ABGjhyp+SDOaO3atZBIJBg0aBDi4uKwbt06TJo0CXv27AEAjBgxArGxsQgNDcWUKVMAABYWFgAAtVqNkSNH4vr16+jRowfKly+P+/fvY/Pmzfjnn3+wYsUKAMDUqVNx5coV+Pv7Y9euXZDJZNi5cyd++eUXzJs3D46Ojqhbty58fX2xdetWjBgxAuXKlQMAlC9fPtv+ePDgAXr16gVHR0cMHToU5ubmOH78OEaPHo2lS5eidevWKFasGL7++muMHz8eW7duRb9+/aBWqxEQEAALCwvMnDlTU9+JEyeQlJSEXr16wcbGBrdu3cK2bdsQGhqKJUuWaB1bpVJh6NChqFOnDiZNmoQjR47gm2++gZmZGRYtWoQOHTqgTZs22LlzJ/z9/eHh4YHSpUtr1fHNN99AoVBgzJgxePz4MX766Se8ePECW7duhUQiybLNue3zBw8eYPjw4XBzc8O4ceNgbGyMJ0+e4Pfff8/xNUZEZJAEKtL27dsnuLq6Ck+fPtWsa968ueDq6ir89ttvmnXh4eGCu7u7MHfu3PfWOXnyZKFSpUrCrVu3Mm1Tq9WCIAjC//73v0zHiIuLE1q0aCE0b95cUKlUgiAIwpUrVwRXV1fB29tbSE5O1pTdvHmz4OrqKty7d0+zbtiwYULz5s0zHfPgwYNCpUqVtI4lCILw008/Ca6ursL169c16y5cuCC4uroKK1asEP7991/Bw8NDGDVqlNZ+x48fF1xdXYUrV668ty8EQRD69+8vtG/fXit+tVot+Pj4CG3atNEqO2HCBKFGjRrC48ePhXXr1gmurq7Czz//rFUmMTEx0zFWr14tuLm5Cc+fP9es8/f3F1xdXYVVq1Zp1kVHRwvVq1cX3NzchGPHjmnWP3r0SHB1dRWWLFmiWZf+2ujSpYuQkpKiWb927VrB1dVVOHXqlGZd3759hb59+2qWc9vnGzduFFxdXYXw8PBseo+IqPDg8B9lqUKFCqhTp45m2c7ODmXLlsXTp09z3E+tVuPUqVNo3rw5qlWrlml7+pWN4OBgVK9eXesYFhYW8PHxwfPnz/Hw4UOt/bp27QpjY2PNcvp+74sHSLuyU758eZQrVw4RERGaf15eXgCgNXTXqFEj+Pj4YPny5Rg7dixMTEzwzTffvPcY2YmKisKVK1fg7e2NuLg4zbEjIyPRqFEj/PPPPwgLC9OUnz59OiwtLTFu3DgsXrwYnTp1QqtWrbTqNDU11fyckJCAiIgI1KxZE4Ig4O+//84UQ/fu3TU/KxQKlC1bFmZmZvD29tasL1euHBQKRZb96ePjAyMjI81yr169IJfLERwcnG27c9vnCoUCAHD69Gmo1eps6yMiKgw4/EdZKlmyZKZ11tbWmrk0KpUKERERmbbHxMQgLi7uvUOEL168QI0aNTKtTx9Oe/HiBVxdXTXrS5UqpVUu/cM4JibmvW158uQJHj16hPr162e5PTw8XGvZ398fZ86cwZ07d7BgwQIUK1bsvcfIzr///gtBELB48WIsXrw42+M7OjoCAGxsbDBt2jSMHz8e9vb2mDZtWqbyL168wJIlS3DmzJlMc5vi4uK0lk1MTGBnZ6e1zsrKCiVKlMg0dGdlZZVlf37yySdayxYWFnBwcMjx24+57fPPPvsMe/bswbRp07BgwQLUr18frVu3xqeffgqplH/zEVHhwqSKsvS+bwT+999/aNmypda6LVu25Di3KD+y+4AVMkzqzo5arYarq6tmrtW7SpQoobV8584dzYf+/fv38xhp5mMDwKBBg9C4ceMsy7i4uGgt//LLLwCA6OhohIaGahJIIC2ZHThwIKKjozFkyBCUK1cO5ubmCAsLQ0BAQKarPdmdx+zW56Y/cyO3fW5qaort27fj6tWrOHfuHC5cuIDAwEDs2rULGzZs4DdTiahQYVJFOnFwcMDGjRu11lWqVAlWVlawtLTEgwcPcty/VKlSePz4cab1ISEhmu15ld2kaRcXF9y9exf169fPtky6hIQETJkyBRUqVEDNmjWxbt06tGrVCtWrV3/vcbKSPunbyMgIDRo0eG/58+fPY8+ePRgyZAiOHDmCgIAA7N69G3J52lv1/v37+Oeff/D999+jc+fOmv0uXryY65jy6smTJ5phOwCIj4/Hq1ev0KRJk2z3yUufS6VS1K9fH/Xr18eUKVOwatUqLFq0CFevXs1VnxERGQpeXyedmJiYoEGDBlr/rK2tIZVK0apVK5w9exZ//vlnpv3Sr4Q0bdoUt27dwo0bNzTbEhISsHv3bjg5OaFChQp5jsnMzAyxsbGZ1nt7eyMsLAy7d+/OtC0pKQkJCQma5fnz5+O///7D3LlzERAQACcnJwQEBGjdBsDMzAwAsjzWu4oVK4Z69eph165dePnyZabtGYdQY2JiMG3aNFSvXh0TJkzA7Nmzcfv2baxatUpTJv2KXcYrSoIgFOhNT3ft2oXU1FTN8k8//QSlUpljUpXbPo+Kisq0vXLlygCQ6dYLRESGjleqSHQTJkzAxYsX4evrq/k6/atXr3DixAns2LEDCoUCw4YNw7FjxzB06FD4+vrC2toaBw8exLNnz7B06VKd5tNUrVoVgYGB+O6771CtWjWYm5ujRYsW6NSpE44fP46ZM2fi6tWrqFWrFlQqFUJCQnDixAmsW7cO1apVw+XLl7Fjxw6MGTMGVatWBQB899138PX1xY8//ojJkycDSPvQl8lkWLt2LWJjY2FsbAwvL69s517NnDkTvXv3RocOHdCjRw+ULl0ar1+/xh9//IHQ0FAcPnwYAPC///0PUVFR2LhxI2QyGZo0aYLu3btj1apVaNWqFSpVqoRy5crBxcUF33//PcLCwmBpaYmgoKBczS3TVWpqKgYMGABvb288fvwYO3bsQO3atTMN/2aU2z5fvnw5rl27hqZNm8LJyQnh4eHYsWMHSpQogdq1axdYm4iICgKTKhKdo6Mjdu/ejcWLF+PIkSOIi4uDo6MjmjRpovnmmr29PXbu3IkffvgB27ZtQ3JyMtzc3LBq1So0a9ZMp+P27t0bd+7cwf79+7Fp0yY4OTmhRYsWkEqlWL58OTZt2oRDhw7h559/hpmZGZydneHr64uyZcsiLi4OU6dORZUqVTBixAhNnXXq1EG/fv2wceNGtGnTBh4eHnBwcMCsWbOwevVqTJ06FSqVClu2bMk2qapQoQL27duHZcuW4cCBA4iKioKdnR2qVKmC0aNHA0j79tvBgwcREBCgNS8tICAAly5dgr+/P/bu3QsjIyOsWrUKs2fPxurVq2FiYoLWrVujT58+6NSpk0799j4zZszAkSNHsGTJEqSmpqJdu3aYNm1ajsN6uelzIO1Gs8+fP8e+ffsQGRkJW1tb1KtXD2PHjoWVlVWBtIeIqKBIBLFmphLRR2X//v2YMmUK9u7dm+XtMYiISBvnVBERERGJgEkVERERkQiYVBERERGJgHOqiIiIiETAK1VEREREImBSRURERCQCJlVEREREImBSRURERCQCJlVEREREImBSRURERCQCJlVEREREImBSRURERCSC/wOyDLAehW2zaAAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 400x300 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "metrics = collect_results(run_path, task)\n",
        "_, conf = get_model_from_run(run_path, only_conf=True)\n",
        "n_dims = conf.model.n_dims\n",
        "\n",
        "models = relevant_model_names[task]\n",
        "print(models)\n",
        "print(metrics)\n",
        "basic_plot(metrics[\"standard\"], models=models)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "PhkgGW1WplF4"
      },
      "outputs": [],
      "source": [
        "# plot any OOD metrics\n",
        "for name, metric in metrics.items():\n",
        "    if name == \"standard\": continue\n",
        "\n",
        "    if \"scale\" in name:\n",
        "        scale = float(name.split(\"=\")[-1])**2\n",
        "    else:\n",
        "        scale = 1.0\n",
        "\n",
        "    trivial = 1.0 if \"noisy\" not in name else (1+1/n_dims)\n",
        "    fig, ax = basic_plot(metric, models=models, trivial=trivial * scale)\n",
        "    ax.set_title(name)\n",
        "\n",
        "    if \"ortho\" in name:\n",
        "        ax.set_xlim(-1, n_dims - 1)\n",
        "    ax.set_ylim(-.1 * scale, 1.5 * scale)\n",
        "\n",
        "    plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wx02SWrspoDT"
      },
      "source": [
        "Prompting\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "a2eawlGRpqTz"
      },
      "outputs": [],
      "source": [
        "model, conf = get_model_from_run(run_path)\n",
        "\n",
        "n_dims = conf.model.n_dims\n",
        "batch_size = conf.training.batch_size\n",
        "\n",
        "data_sampler = get_data_sampler(conf.training.data, n_dims)\n",
        "task_sampler = get_task_sampler(\n",
        "    conf.training.task,\n",
        "    n_dims,\n",
        "    batch_size,\n",
        "    **conf.training.task_kwargs\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "6q_tsQfPpsVU"
      },
      "outputs": [],
      "source": [
        "task = task_sampler()\n",
        "xs = data_sampler.sample_xs(b_size=batch_size, n_points=conf.training.curriculum.points.end)\n",
        "ys = task.evaluate(xs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "lHXcz73ipvAi"
      },
      "outputs": [],
      "source": [
        "with torch.no_grad():\n",
        "    pred = model(xs, ys)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 458
        },
        "id": "6hundlcKpwDf",
        "outputId": "3ecaca97-2bfa-40f3-ca4a-e95e66272874"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkEAAAG5CAYAAACJLeBEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9PklEQVR4nO3dd3hTZd8H8G9Gd5sOaMsopbTQUmihbEqhsrcsWQ5ARQEFWfI84HhRxAcRcQEiCCiIylTEUQq4AAEB2Xu0UCijhY6ku01y3j/SHhq607RZ3891eck5uc/pfefknPxyT4kgCAKIiIiIbIzU1BkgIiIiMgUGQURERGSTGAQRERGRTWIQRERERDaJQRARERHZJAZBREREZJMYBBEREZFNYhBERERENolBEBEREdkkuakzYM4EQYBWWzMTakulkho7tzlg+SyftZeR5bN81l5Gls/w80okkkqlZRBUDq1WQGpqltHPK5dL4enpApUqG2q11ujnNzWWz/JZexlZPstn7WVk+Qzn5eUCmaxyQRCbw4iIiMgmMQgiIiIim8QgiIiIiGwSgyAiIiKySQyCiIiIyCYxCCIiIiKbxCCIiIiIbBKDICIiIrJJDIKIiIjIJplVEJSQkID58+dj6NChaNGiBQYPHlyp4wRBwBdffIHu3bujVatWGDNmDE6dOlWzmSUiIiKLZlZB0NWrV7Fv3z40btwYQUFBlT5uzZo1WLZsGZ599lmsXr0a3t7eeP7553Hr1q0azC0RERFZMrMKgnr27Il9+/Zh2bJlaNmyZaWOycvLw+rVq/H888/j2WefRWRkJD766CN4eHhg3bp1NZxjIiIislRmtYCqVFr1mOzEiRPIzMzEgAEDxH329vbo06cP9u7dW638CIKAgoKCUl+TSCSQyx++fWWl06UF5HK7YmnzkZ9vh4KCfKjVQgVpyz4vANjZGZZWrS6AUM7ivYanVUMQUGb5SqYt+8RyuVxcCVijUZe72rDhaTXQastevK+0tIIgKbV8MplM/AxXdN7aSKvVaqHRaMpMK5VKIZPJSk37aBnLS1uV85aXVhAEqNVqI6WVQCaTl5m2ePm0WpSbtriq3ffVeUZUL+3D8hVAIpFV6rxAbT4jyk5c2bSCIIEgOIvb5vSMKEtV7mWp9OH7YI7PiOqmlcsf5qG8z6Whz4jKMqsgyBDx8fEAgMDAQL39QUFB2LBhA3Jzc+Ho6GjQuTMyVFizZnmprwUENMGQIU+I21988XmZF6dhQz888cRYcXvNmjXIyckpNa2Pjy/Gjh0nbm/cuAEZGapS03p51cEzzzwnbm/e/B1SU1NKTevmpsBzz00St7dv34rk5KRS0zo6OmHSpKni9s6dO3D7dmKpaeVyOV5+eaa4HRPzM27cuF5qWgCYPn2O+O89e2Jx7dqVMtO+9NJ0yOX2AIA///wdFy+eLzPtCy+8DGdn3QPxwIH9OHv2VJlpn332RSgU7gCAf/7ZjxMn/i0z7dNPP4s6deoCAP799zCOHj1cZtoxY56Gr299AMDp0//i4MH9ZaYdMWI0/Pz8AQDnz5/Gvn2/l5n28ceHo0kTXfPwlSsX8NtvsWWmHTDgcTRrFgJA17y8a9fPZabt3bs/WrQIAwBcv34dP/+8o8y0jz3WC61btwEAJCYm4ocftpaZNioqGu3adQQAJCUlYcuWb8tM27FjJDp3jgIApKQ8wLffri8zbdu27dG1a3cAgEqlxPr1a8pMGx4egR49egMAsrOzsXbtyjLThoa2RJ8+uh9RBQX5Zd7zANC0aTAGDhwibq9cWXba6jwjvvxyLXJz+Yyo6Bnx2muvQSbTfZla2zNi5MixqFMnBDKZFJcunbO6Z0THjp0BACkp97Fp08Yy0xr6jKgsiw+CVCoV7O3t4eDgoLdfoVBAEAQolUqDg6Dy2NnJ4enpIm4X/RIojVwuMzitVFp2WplMqpe26GFQGqlUopdWLi87Wq5KWolEP62dXfkfqeJp7e3LT+vh4QJ7e/tKpnWGi4vu3A4O5ad1d3eGh0dRWrty0yoUTmKenZzsy03r5mZYWmfn8tO6ujqKaV1cHMpN6+LiYFDa+/fLv0ecne3FtEqlU7lpnZweps3OrnzagoKsctM6ONiJaSWS8ms0HBwe3p/25b+9sLd/mDY/v/zPQ/G0FanOM6K8+57PCH0KhVOl0lraM6Lo/lUonKzyGVF03SrKg6HPiMqSCOXVS5rQvHnzcO7cOfzyyy/lpvv888+xcuVKnD17Vm9/bGwsZsyYgf3798PX19egPKjVGqSmlv4LSyKRPlLVnV/meXTV4g9vJK1WDTc3J2Rk5ECj0ZabVldNWNYlkpTSHFa5tLrq6/KqpO0NTKuGVIoyy/doWkEor5rZTvziqKm0uqrjsqttS0srk0lLLZ9MJn+kSrrs89ZGWl2VdHlVx7JHqq8fpn20jOWlrcp5y0urq+our1q88mmL35+lpS1ePq0W5aYt67xARfe94c+IqqUted8/LF8upNJHm8NM/4wo7/6sbFqZTAovLwUyMnKh0WjN6hlRlqrcy/b29vDwcIFKlYP8/AKze0ZUN629vR0UCiekp2chP7/sz7shzwiFwqncgL84i68JUigUyM/PR15enl5tkEqlgkQigbu7u8HnlkgkkEjKfovUam2xtOW/lcXTyuVy2NvbQyotKLXtVv+85bdvGpoWkKGcH6bVSCuFVCots3yPppVIyv6gajQCHj6waypt+de4tLRllU+rRbHt8s9bO2nL/1wKQtmf4UfLWF7aqpy3ttIC5ad9tHzGOq9p0pa874uXzxyfEeXdn5VNK5VKIZFICgMgbYXnrc1nRFmqci8XxZQaja4fork9I6qbtugHpCDUXB4qw6xGhxmiqC/Q9ev6bczx8fFo0KBBjTSFERERkeWz+CCobdu2cHV1xa5du8R9BQUF2LNnD6Kjo02YMyIiIjJnZtUclpOTg3379gEAbt++jczMTMTG6nq5d+zYEV5eXpgwYQLu3LkjDn93cHDA5MmTsXz5cnh5eSE4OBibNm1Ceno6Jk6caLKyEBERkXkzqyAoJSUFM2bM0NtXtP3111+jU6dOpc4/8OKLL0IQBHz55ZdITU1FaGgo1q1bh0aNGtVa3omIiMiymFUQ5Ofnh8uXL5ebZuPGkvMJSCQSTJ48GZMnT66prBEREZGVsfg+QURERESGYBBERERENolBEBEREdkkBkFERERkkxgEERERkU1iEEREREQ2iUEQERER2SQGQURERGSTGAQRERGRTWIQRERERDaJQRARERHZJAZBREREZJMYBBEREZFNYhBERERENolBEBEREdkkBkFERERkkxgEERERkU1iEEREREQ2iUEQERER2SQGQURERGSTGAQRERGRTWIQRERERDaJQRARERHZJAZBREREZJMYBBEREZFNYhBERERENolBEBEREdkkBkFERERkkxgEERERkU1iEEREREQ2iUEQERER2SQGQURERGSTGAQRERGRTWIQRERERDaJQRARERHZJAZBREREZJMYBBEREZFNYhBERERENolBEBEREdkkBkFERERkkxgEERERkU1iEEREREQ2iUEQERER2SQGQURERGSTGAQRERGRTWIQRERERDaJQRARERHZJAZBREREZJMYBBEREZFNYhBERERENolBEBEREdkkBkFERERkkxgEERERkU1iEEREREQ2iUEQERER2SQGQURERGSTzC4IiouLw3PPPYeIiAhERUVhyZIlyM/Pr/C4tLQ0zJ8/H927d0dERAQGDx6MTZs21UKOiYiIyBLJTZ2B4pRKJSZMmICAgAAsX74cSUlJWLx4MXJzczF//vxyj50xYwbi4+Mxe/Zs1K9fH/v378fbb78NmUyG0aNH11IJiIiIyFKYVRC0efNmZGVlYcWKFfDw8AAAaDQaLFiwAJMnT4avr2+px92/fx9HjhzBe++9hxEjRgAAIiMjcfbsWfz6668MgoiIiKgEs2oO279/PyIjI8UACAAGDBgArVaLgwcPlnmcWq0GALi5uentd3V1hSAINZJXIiIismxmFQTFx8cjMDBQb59CoYC3tzfi4+PLPK5+/fro2rUrVq1ahWvXriEzMxMxMTE4ePAgnn766ZrONhEREVkgs2oOU6lUUCgUJfa7u7tDqVSWe+zy5csxa9YsDBo0CAAgk8nw5ptvol+/ftXKk1xu/DhRJpPq/d/asHyWz9rLyPJZPmsvI8tXO8wqCDKUIAh47bXXcOPGDXz44Yfw9vbGoUOHsGjRIri7u4uBUVVJpRJ4eroYObcPKRRONXZuc8DyWT5rLyPLZ/msvYwsX80yqyBIoVAgIyOjxH6lUgl3d/cyj/vrr78QGxuLn376CSEhIQCATp06ISUlBYsXLzY4CNJqBahU2QYdWx6ZTAqFwgkqVQ40Gq3Rz29qLJ/ls/YysnyWz9rLyPIZTqFwqnQNk1kFQYGBgSX6/mRkZOD+/fsl+goVd+3aNchkMgQHB+vtDw0NxbZt25CTkwMnJ8OiTbW65j58Go22Rs9vaiyf5bP2MrJ8ls/ay8jy1SyzamyMjo7GoUOHoFKpxH2xsbGQSqWIiooq87iGDRtCo9Hg8uXLevvPnz+POnXqGBwAERERkfUyqyBo7NixcHFxwdSpU/H333/j+++/x5IlSzB27Fi9OYImTJiAPn36iNvR0dFo0KABpk+fjp07d+Lw4cP44IMPsGPHDjzzzDOmKAoRERGZObNqDnN3d8eGDRuwcOFCTJ06FS4uLhg5ciRmzZqll06r1UKj0Yjbrq6uWL9+PT7++GMsXboUGRkZ8PPzw7x58xgEERERUanMKggCgKCgIKxfv77cNBs3biyxr3Hjxvjkk09qJlNERERkdcyqOYyIiIiotjAIIiIiIpvEIIiIiIhsEoMgIiIiskkMgoiIiMgmMQgiIiIim8QgiIiIiGwSgyAiIiKySQyCiIiIyCYxCCIiIiKbxCCIiIiIbBKDICIiIrJJDIKIiIjIJjEIIiIiIpvEIIiIiIhsEoMgIiIiskkMgoiIiMgmMQgiIiIim8QgiIiIiGwSgyAiIiKySQyCiIiIyCYxCCIiIiKbxCCIiIiIbBKDICIiIrJJDIKIiIjIJjEIIiIiIpvEIIiIiIhsEoMgIiIiskkMgoiIiMgmMQgiIiIim8QgiIiIiGwSgyAiIiKySQyCiIiIyCYxCCIiIiKbxCCIiIiIbBKDICIiIrJJVQ6CcnJyMG3aNPz00081kR8iIiKiWlHlIMjJyQmHDh1Cbm5uTeSHiIiIqFYY1BzWrl07nDx50th5ISIiIqo1BgVB8+fPx/Hjx/Hxxx/j3r17xs4TERERUY2TG3LQkCFDoNFo8MUXX+CLL76ATCaDvb29XhqJRILjx48bJZNERERExmZQENSvXz9IJBJj54WIiIio1hgUBC1evNjY+SAiIiKqVZwniIiIiGySQTVBAJCZmYn169fjr7/+wp07dwAADRo0QPfu3fHss8/C1dXVaJkkIiIiMjaDaoKSkpIwbNgwrFixAtnZ2Wjbti3atm2LnJwcrFixAsOHD0dycrKx80pERERkNAbVBC1duhQPHjzA6tWr8dhjj+m9tm/fPsycORMffvgh3n//faNkkoiIiMjYDKoJOnDgACZMmFAiAAKAxx57DOPGjcO+ffuqnTkiIiKimmJQEJSTk4M6deqU+XrdunWRk5NjcKaIiIiIappBQVBQUBB+/fVX5Ofnl3itoKAAv/76K4KCgqqdOSIiIqKaYlCfoBdffBGzZs3CqFGj8NRTTyEgIAAAcP36dWzevBmXL1/Gxx9/bMx8EhERERmVQUHQgAEDkJOTgw8//BBvvfWWOHu0IAioU6cOFi1ahP79+xs1o0RERETGVOUgSBAEZGVlYdCgQRgyZAjOnTunN09QWFgY5HKDpx8iIiIiqhVVjlYKCgrQsWNHzJo1Cy+++CIiIiIQERFRA1kjIiIiqjlV7hhtb2+PunXrllg1noiIiMiSGDQ6bPjw4di5c2epo8OIiIiILIFBnXdCQkLw+++/Y/DgwRg+fDgaNmwIR0fHEun69u1b7QwSERER1QSDgqDZs2eL//70009LTSORSHDx4sUqnzsuLg7vvvsuTp48CRcXFwwdOhQzZ86sVPNbUlISPvroI+zbtw/Z2dlo2LAhXnrpJQwZMqTK+SAiIiLrZlAQ9PXXXxs7HwAApVKJCRMmICAgAMuXL0dSUhIWL16M3NxczJ8/v9xjk5OTMWbMGDRp0gQLFy6Eq6srrl69yiY7IiIiKlWVg6C8vDxcunQJoaGh6NChg1Ezs3nzZmRlZWHFihXw8PAAAGg0GixYsACTJ0+Gr69vmcd+8MEHqFevHtauXQuZTAYAiIyMNGr+iIiIyHpUuWO0g4MDli5diuvXrxs9M/v370dkZKQYAAG6iRm1Wi0OHjxY5nGZmZnYtWsXnnrqKTEAIiIiIiqPQc1hzZo1w+3bt42dF8THx+OJJ57Q26dQKODt7Y34+Pgyjzt//jwKCgogl8vxzDPP4OTJk/Dw8MCwYcMwc+ZM2NnZGZwnudygAXTlksmkev+3Niyf5bP2MrJ8ls/ay8jy1Q6DgqBZs2bh1VdfRadOndClSxejZUalUkGhUJTY7+7uDqVSWeZxDx48AAC8+eabGD16NKZNm4YzZ85g2bJlkEqlePXVVw3Kj1Qqgaeni0HHVoZC4VRj5zYHLJ/ls/YysnyWz9rLyPLVLIOCoG+++QYeHh6YOHEi/Pz84OfnBwcHB700EokEn3/+uVEyWRGtVgsA6NKlC+bNmwcA6Ny5M7KysvDll19i6tSppQ7hr/i8AlSqbKPmFdBFvgqFE1SqHGg0WqOf39RYPstn7WVk+SyftZeR5TOcQuFU6Romg4KgK1euAADq168PjUaDhISEEmmKFlWtCoVCgYyMjBL7lUol3N3dyz0O0AU+xUVGRmLVqlVISEhASEhIlfMDAGp1zX34NBptjZ7f1Fg+y2ftZWT5LJ+1l5Hlq1kGBUF//PGHsfMBAAgMDCzR9ycjIwP3799HYGBgmcc1bdq03PPm5eUZJX9ERERkPcxquffo6GisWrVKr29QbGwspFIpoqKiyjyuYcOGCA4OxqFDh/DMM8+I+w8dOgRHR8cKgyQiIiqfVquFRqM2dTZEWq0Eubky5OfnQaMRTJ0do2P5SieTySGVGq8ztcFBkEajQWxsLI4cOYKUlBRMnz4dISEhyMjIwOHDh9G2bVvUrVu3SuccO3YsNm7ciKlTp2Ly5MlISkrCkiVLMHbsWL05giZMmIA7d+5g79694r5Zs2bh5Zdfxv/+9z90794dZ8+exZdffomJEyfC2dnZ0GISEdk0QRCgUqUiJyfT1Fkp4cEDqdgn1BqxfKVzcnKFQuFlULebRxkUBKlUKrzwwgs4c+YMnJ2dkZOTI9bAODs7491338WwYcP0lteoDHd3d2zYsAELFy7E1KlT4eLigpEjR2LWrFl66XS/SDR6+3r27ImPPvoIK1euxKZNm+Dj44NXXnkFkyZNMqSIREQEiAGQq6sn7O0djPLFYywymcQqa0mKsHz6BEFAfn4eMjPTAADu7nWqnQeDgqClS5fi6tWrWLduHUJDQ/WGyctkMvTr1w/79u2rchAEAEFBQVi/fn25aTZu3Fjq/oEDB2LgwIFV/ptERFSSVqsRAyBX15LTl5iaXC616k7DLF9J9va6keiZmWlwc/OsdtOYQUf//vvvGDduHKKiokr9VRAQEFAjkykSEVHtKapxL/riITIHRZ9HY/RRMygIysjIgJ+fX5mvq9XqEs1VRERkmcypCYzImJ9Hg4Igf39/nD9/vszXDx48iKCgIIMzRURERFTTDOoTNHLkSCxduhSdOnUSJyiUSCTIz8/HZ599hgMHDuCdd94xakaJiIgM0bVr+wrTvP76Wxg48PFayE3Z7ty5jffffxcXLpxHTk42vv56EwIDm5k0T9bOoCBowoQJuHbtGmbPni3O5zNnzhykp6dDrVZjzJgxGDVqlFEzSkREZIhVq77S254y5TmMHDkGvXv3F/c1bFh2F4/asnbtKty5cxvvvvs+XFxc4e/vb+osWT2DgiCJRCIOg9+9ezcSEhKg1Wrh7++PAQMGoEOHDsbOJxERkUHCwsJL7PPxqVfq/iJ5eblwcKj6mpPVkZBwA61aRaBTp0gA1RsdZor8A0B+fj7kcuNOaFiTqjVjdPv27dG+fcXVjEREROZq3brV2Lz5G3z66ef49NMPcfXqZbzwwkt46qlx+Pzz5Th8+G/cvXsHLi6uaN26DV55ZbbeZMDTpk2Cs7MzBgwYjC++WIkHD+4jNLQl5s59U6+GaePG9fjllx9x/34ynJ2dERQUjLlz34BEIsGoUUMAAJcvX8Tu3TGoV68+fvzxVwDAjz9+jy1bvsW9e3dRp05dDB48FOPHPy8GGjExP2PRogVYtepLrF27CufOncHAgY+je/demD59Cj78cDl++WUn/vnnINzcFJgy5RX07dsf27ZtxqZNG5GTk4PHHuuB2bPnwt7eXsxvcnISVq1agSNHDiEnJxehoS3wyiuz0bx5qJhm5MjH0aVLV/j61sMPP2xDcnISfv55Lzw8PGrykhmNWS2bQUREZAoFBQVYsOBNjB79FCZPngqFQrdod1paKsaNew5163ojPT0Nmzd/i2nTJuGbb7ZCLn/4FXr16hWkpW3ElCmvQKvVYPnyj/HOO/+H1at1TXG7dv2CtWs/xwsvTEHLluHIysrE6dOnkJWVhcaNA7Bq1Vd499230KhRI0yY8ALs7e0AANu3b8YnnyzFyJFj0KVLN5w9expffbUGmZmZmDZtpl4ZFix4E0OGDMf48c/DwcEReXm5AIClSxdj4MDBGDJkGH766Ue8++58XLt2Bdevx+E//3kNd+7cxvLlH6NBg4YYP/55ALpJkV9++QU4OTlh5sz/wNXVFdu3b8WMGVOwefMOeHp6iX93374/4Ofnjxkz5kAqlcLJqfZroAzFIIiIiKrk2KVk/HggHrn5ppkKxdFehuHdAtE5rJ7RzqlWqzFp0svo1auv3v7XX39L/LdGo0FYWCsMHz4QJ078i44dO4uvZWZm4Msvv4WnpycAICcnB4sWLUBychJ8fHxx8eJ5BAU1w7hxz4nHdOvWXfx3WFg4HB0d4eHhKTbTaTQarF+/Fr169cXMmf8BAHTs2BlqtRqbN3+DceOehbu7h3iOoUNH4JlnnhW3T5z4FwDQo0cvPPfciwCA0NAw7N//J377bTe2bt0pBnInTx7Hn3/+JgZB27ZtQmZmBtas2SAGPO3adcSTT47Apk0b8fLLM/Teu6VLl8HJyakK77h5YBBERERVEnskAXdTsk2ah11Hbho1CAKAyMiuJfYdPnwQGzasw/XrccjKyhL337qVoBcENW0aLAZAABAQ0AQAkJycDB8fXwQHN8eOHduxfPlHiI7uiZYtw/RqkkqTkHAD6enp6Nmzt97+nj37YOPGr3DhwnlERj5cXLxLl5L5B4AOHTqJ/3Z1dYWHhyciItrq/f1GjRrj5Mnj4vbRo/+gTZv2cHNTQK3WTUoolUoREdEWFy9e0Dt/mzbtLDIAAhgEERFRFQ3o1Bg7TFwTNKCTcUdOOTo6llhs++LF85g3bza6dXsMzzwzAR4eukU7J09+Fnl5+Xpp3dzc9Lbt7HTNWfn5eQCAgQMfR3Z2Nn76aQe2bPkOrq6u6N9/MF56aVqZHZhVKhUA6DU9AYCXl247I0Olt9/Ts/S1tErLm6urq94+uVyO/PyHZVIq03H+/Fl0794Zj3p0JF1Zf9cSMAgiIqIqad/cB+2b+5g6G0ZV2izE+/f/BVdXV7zzzmKxE/K9e3cNOr9UKsXo0U9i9Ogncf9+Mn77bQ9WrVoODw8PPPvsC6Ue4+5e1C8pTW9/amoqAMDNTX89N2POpOzmpkCnTl3w4otTSrxmZ2evt23JE4ozCCIiIipFXl4u5HK5XnCxZ8+uap/X29sHTz75DPbujcWNG9fLTOfv3xgeHp7488/f8NhjPcT9f/yxF3Z2dmjRomW181KW9u07Ys+eXWjcuInFNnVVRqWCoNdee63KJ5ZIJFi0aFGVjyMiIjIHHTp0wtatm/Dxx0sQHd0D586dwe7dMQada8mS/8HNTYGWLcPh5uaGs2dPIy7uKkaMGFnmMTKZDM8+OxGffLIUnp5eiIyMwvnzZ/Hdd19j1Kgn9TpFG9vYsU9j795YTJs2CaNGjYWvbz2kp6fhwoXzqFu3LsaMebrG/nZtqlQQdOTIkRL7cnNzxSq5oio7pVIJQNdeac2RIxERWb/IyK546aVX8P33WxET8zPCw1tjyZJP8OSTI6p8rvDw1vjppx34+ecfkZubiwYNGuKVV2Zh8OBh5R43cuRYyOVybN78HXbs2IY6deriuedeFEdx1RR3dw+sXv0V1qz5HJ9/vhwqlRKenl5o0SIM0dHda/Rv1yaJIAhCVQ+6du0ann/+eQwfPhwTJkwQO2mlpqZiw4YN+PHHH/Hll19a/CKqGo0WqalZFSesIrlcCk9PF6SlZRk8G6g5Y/ksn7WXkeWrnIKCfKSk3EWdOvVL9AMxB9WZUdkSsHylq+hz6eXlApmscjNWGzSv9cKFCxEdHY1Zs2aJAZDuD3th1qxZ6NatGxYuXGjIqYmIiIhqhUFB0OnTp9GiRYsyXw8NDcXp06cNzhQRERFRTTMoCHJ3d8f+/fvLfH3//v0l5iUgIiIiMicGBUFjxozBX3/9hZdeegmHDh1CYmIiEhMTcfDgQUyZMgX79+/H2LFjjZ1XIiIiIqMxaJ6gl19+Gfn5+Vi3bh3++usvvddkMhkmTZqEl19+2Rj5IyIiIqoRBk+WOHPmTIwfPx6HDh3CnTt3AAANGzZEZGSkXmdpIiIiInNUrRmjvby8MHjwYGPlhYiIiKjWGBwEaTQaxMbG4siRI0hJScH06dMREhKCjIwMHD58GG3btkXdunWNmVciIiIiozEoCFKpVHjhhRdw5swZODs7IycnB8888wwAwNnZGe+++y6GDRuG2bNnGzWzRERERMZi0OiwpUuX4urVq1i3bh1+++03FJ90WiaToV+/fti3b5/RMklERERkbAYFQb///jvGjRuHqKgovdV1iwQEBOD27dvVzhwRERGV7+rVy1i3bjVyc3P19sfE/IyuXdsjPT291vKydet3OHz471r7e9VlUBCUkZEBPz+/Ml9Xq9XQaDQGZ4qIiIgq5+rVK/jqqzUlgqDIyK5YteoruLq61lpetm7dhMOHD9ba36sug/oE+fv74/z582W+fvDgQYtfPJWIiKgieXm5cHBwNHU2SuXp6QlPT09TZ6Naavr9NagmaOTIkfj+++8RExMj9geSSCTIz8/Hxx9/jAMHDmDMmDFGzSgREZEh7t69g65d25f6X0zMz2K6c+fOYPr0Kejduyv69XsMb7/9BtLSUkucJybmZ7z//rsYOLAXXnxxAgBApVJi0aIFGDSoF3r2jMKUKc/j1KkTlcpfTMzPmDBhLHr27IJhwwZg9erP9FpTMjIy8P7772LYsAHo2bMLRowYhLfeek08dtGiBQCAwYN7o2vX9hg58nHxteLNYUX5j439FR98sAj9+3fH4MF9sHnzNwCA337bjSefHIG+fR/D66//BxkZGWIecnJy8NFH7+PJJ0egV68ojBz5OD74YBEyMzPFNCNHPo579+7ihx+2lXh/tVot1q9fi5EjH0ePHpF46qknsGPHdr33Yd261ejTpxsuXDiHyZOfQ8+eXfD999sq9R4ayqCaoAkTJuDatWuYPXs2FAoFAGDOnDlIT0+HWq3GmDFjMGrUKKNmlIiIzEdBQUGZr0kkEsjl8kqmBeRyO4PSVladOnWxatVXevtiY3/BTz/tgJ9fIwC6AOiVVyajc+coLFjwHnJzc7BmzeeYN+9VrF6tf+zq1SsQGdkVb7/9P2i1Wmg0Grz66nTcvXsbL730Cjw962D79s2YNWsqPv/8SzRvHlpm3jZv/gaff74co0c/hWnTZuLGjRv44ouV0Gq1eOWVGQCA5cs/wpEjhzBlyiuoV68+UlIe4J9/DgHQNXlNmDARGzasw4cfLoeLiyvs7ct/j774YiW6d++JhQsXY//+fVix4hOkp6fj5MnjePnl6cjKysInn3yAlSuXYe7cNwAAubm50Gq1mDTpZXh4eCI5OQlff/0lXnvtVSxfvhoAsGjRB/jPf2YgPDwCY8fqRow3bKjrOvPZZ59i+/bNGD/+eYSHt8ahQwfw/vuLkJ9fgCeeeFhpUlBQgAUL3sTo0U9h8uSpUCjcy7+41WRQECSRSMRh8Lt370ZCQgK0Wi38/f0xYMAAdOjQwdj5JCIiM7JmzfIyX/P3b4LBg4eL21999TnUanWpaRs08MOwYaPF7Y0b1yI3N6fUtN7evhg16ukq59Xe3h5hYeHi9tmzp/Hrrz9h4sTJaNUqAgCwatUKNG8eikWLPhAH/AQGNsX48WNw+PDfiIzsKh7ftGkI5s37P3H777/34eLF8/jww+Xo1CkSANCpUyTGjBmGjRu/xP/+90Gp+crOzsK6dV/gqafGY/LkqQCADh06w85OjuXLP8b48RPg4qLAxYvn0bt3fwwY8HBy4t69+wHQNXkVBRohIaHw8PCo8P0IC2uF6dNfBQC0bdsB+/b9ge+/34Lt23+Gu7vu+GvXruKXX3aKQZCnpyfmzHlNPIdarUb9+g3w8ssv4ObNBPj7N0ZwcHPY2dnDy8tL7/1OT0/H999vwZNPjsPEiZMBAB07doZKpcRXX63FsGEjIZPJxPNOmvQyevXqW2E5jKHKQVBOTg7+85//oG/fvhgyZAjat29fE/kiIiIyuuTkJLzxxn8RFdUNEyZMBKCr5Th79jSmTp2h1wzVqJE/fHx8cfHiBb0gqEuXKL1znj59Ci4uLmIABAByuRyPPdYDe/fuLjMvZ8+eQU5ONnr06KUXJLZv3wl5eXmIi4tDq1ZtEBzcHLt2/YI6deqic+dIBAY2rdZ70KFDJ/HfMpkMDRo0hEQiEQMgQFf2zMwMZGdnw9nZGQAQG/srtmz5FomJt5CT8zBQvXXrJvz9G5f59y5cOAe1Wo0ePXrr7e/duy/27InFrVs3ERDQRNxf/L2uaVUOgpycnHDo0CFER0fXRH6IiMgCvPjiK2W+9ujUKc8991I5afW3x417odJpqyovLxevvTYHHh4eeOONBeL+jAwVNBoNli37CMuWfVTiuOTkJL1tT886etsZGSp4epZcM9PTsw5UKmWZ+VEq0wEAzz//TKmvJyXdAwDMmvVfKBSrsWXLN1i58lP4+Phi3LjnMHz4yDLPXZ5HR4vJ5XIx0CliZ6drUsvPz4ezszP27fsT7777FoYMGY5Jk16GQuGBlJQHeP31OcjPzyv372VkqACgxLqiRdvF3yNHR8cSealJBjWHtWvXDidPnsTo0aMrTkxERFan6EvSlGmr6r33FuLOndtYu/ZrODk5iftdXd0gkUgwbtxziI7uXuK44jUkQMlgTKFQ6HWgLpKWllJunxY3N12f2v/97wP4+vqWeL1RI7/C/LlixoxXMWPGq4iLu4Zt2zbhww8XIzAwCK1btynz/Mb055+/oVmzYPz3v2+I+06ePF6pY4v6DqelpcLb20fcn5qaWvj6w/eotLkHa5JBo8Pmz5+P48eP4+OPP8a9e/eMnSciIiKj2rhxPf788zcsWLBI7ENTxMnJCWFh4UhIuI7mzVuU+K9+/QblnrtVqwhkZWXh6NF/xH1qtRr79/+FVq1al3lcWFgrODo64v79pFL/7qPBFwAEBTXF9Om6Jalu3LgO4GFn8YpqZKojLy+vRKf0PXtiS6STy+2Qn5+vty80NAxyuRx//vm73v7fftsLT08vNGrkb/wMV5JBNUFDhgyBRqPBF198gS+++AIymQz29vZ6aSQSCY4fr1yUSEREVFPOnDmFNWtWolevvnB2dsG5c2fF1xo29IOnpydefnkGZsx4CfPnv4ZevfrCzc0N9+8n49ixIxg48HG0bVt2/9fIyK4IDW2Jd975P0yZMg1eXnWwffsWpKQ8wLhxz5d5nJubGyZOnIKVK5cjOTkZbdq0g0wmw507iThwYD/ef/8DyOUOeOml59GtWw8EBgZBJpMiNvZX2NnZibVAAQEBAIAfftiGbt26w9HREUFB1es39KgOHTrho4/ex/r1a9GyZTj++ecgjh8/WiJdQEAAjh//F8eO/QM3NwXq128ADw8PPPHEGHz33dewt7dHy5bhOHz4IPbs2YVZs/4jdoo2BYOCoH79+tV6lRUREZEhEhNvQavVYu/eWOzdq1978frrb2HgwMcRHt4aK1euxbp1q/HeewtQUFAAb29ftG/fQRxGXxaZTIalSz/FZ599ipUrlyE3NwfBwc3x0Ucryh0eDwBPPvkMvL29sWXLt/j++y2Qy+Vo2NAPXbp0E2tewsNbY/fuX3Hnzh1IpRIEBjbF++9/LHYmDg5ujuefn4RfftmJ7777Gj4+vti+/efy/myVDR06Anfu3Mb27Vvw3Xcb0bFjZ7z11v8wefKzeukmTZqKDz9cjDfemIvs7Czx/Z06dQbc3Nzw888/YsOGdahXrwHmzn0djz8+wqj5rCqJUHz1U9Kj0WiRmppl9PPK5VJ4erogLS0LarXW6Oc3NZbP8ll7GVm+yikoyEdKyl3UqVMfdnb2FR9Qy+RyqVVevyIsX+kq+lx6eblAJqtcbx+D+gQRERERWTqDmsOK3Lt3DxcuXEBGRgZKq1AaNmxYdU5PREREVGMMCoLy8vIwd+5c7NmzB1qtFhKJRG8NsSIMgoiIiMhcGdQc9tFHH2Hv3r2YOXMmNm7cCEEQsHjxYnz55ZeIjo5G8+bNsXPnTmPnlYiIiMhoDAqCdu/ejREjRmDSpElo2lQ3DM/X1xddunTB6tWr4ebmhm+//daoGSUiItPg+BkyJ8b8PBoUBKWkpKBVq1YAdFNcA9BbR6Rfv37Yu3evEbJHRESmUjR/S01OwkdUVUWfR5msWt2aARjYJ6hu3bpIS0sDoJtp093dHdevXxdfz8zMRF4ebxoiIksmlcrg5OSKzEzd897e3sGs5ojTaiXQaKy3lorl0ycIAvLz85CZmQYnJ1dIpdUf4G5QENSqVSucOHFC3O7RowfWrVsHb29vaLVarF+/HhEREdXOHBERmZZCoVvksigQMidSqRRarfXOo8Pylc7JyVX8XFaXQUHQuHHjEBsbi/z8fNjb22PGjBk4efIk/vvf/wIA/P398cYbb1RwFiIiMncSiQTu7nXg5uYJjUZt6uyIZDIJ3N2doVRmW2VtCctX1nFyo9QAFTEoCGrfvj3at3+4jkr9+vWxa9cuXLlyBVKpFIGBgZDLq99WR0RE5kEqlUIqNZ9Zo+VyKRwdHZGTo7HKWZVZvlrKh7FOJJVK0bx5c2OdjoiIiKhGGRQEHTt2rFLpOnToYMjpiYiIiGqcwX2CKjNC4OLFi4acnoiIiKjGGRQEff311yX2aTQa3L59G1u3boVWq8Wrr75a7cwRERER1RSDgqCOHTuW+dqIESPw1FNP4ejRo4iMjDQ4Y0REREQ1yXjjzIpOKJVi0KBB2LZtm7FPTURERGQ0Rg+CAECpVCIjI6MmTk1ERERkFAYFQXfu3Cn1v0uXLuGbb77BunXr9OYRqoq4uDg899xziIiIQFRUFJYsWYL8/PwqnWP9+vUICQnB5MmTDcoDERERWT+D+gT17NmzzNFhgiAgIiICCxYsqPJ5lUolJkyYgICAACxfvhxJSUlYvHgxcnNzMX/+/Eqd4/79+/jss89Qp06dKv99IiIish0GBUGLFi0qEQRJJBIoFAr4+/ujadOmBmVm8+bNyMrKwooVK+Dh4QFAN+pswYIFmDx5Mnx9fSs8xwcffICePXvizp07BuWBiIiIbINBQdCIESOMnQ8AwP79+xEZGSkGQAAwYMAAvPXWWzh48GCFf/fff//Fb7/9htjYWA7RJyIionLVSMdoQ8XHxyMwMFBvn0KhgLe3N+Lj48s9VqPRYOHChZgyZQp8fHxqMptERERkBQyqCRo/fnyVj5FIJNiwYUO5aVQqFRQKRYn97u7uUCqV5R773XffIScnB88++2yV81Yeudz4caJMJtX7v7Vh+SyftZeR5bN81l5Glq92GBQECYKAe/fu4datW3Bzc0OjRo0AAImJiVCpVPD39y/Rf0cQhOrntgwpKSlYtmwZ3n//fdjbG2+VY6lUAk9PF6Od71EKhVONndscsHyWz9rLyPJZPmsvI8tXswwKgmbMmIGXX34ZCxcuxPDhwyGX606jVqvxww8/YOnSpXjvvffQrl27Kp1XoVCUOr+QUqmEu7t7mcd9+umnCAkJQfv27aFSqcS8qNVqqFQqODs7i3msCq1WgEqVXeXjKiKTSaFQOEGlyoFGozX6+U2N5bN81l5Gls/yWXsZWT7DKRROla5hMigIWrJkCUaMGIFRo0bpn0wux+jRoxEfH4/FixdXedbowMDAEn1/MjIycP/+/RJ9hYq7fv06jh07Vuqq9R06dMCaNWsQHR1dpbwUUatr7sOn0Whr9PymxvJZPmsvI8tn+ay9jCxfzTIoCLp8+TKGDh1a5ut+fn7YtGlTlc8bHR2NVatW6fUNio2NhVQqRVRUVJnHvf7662INUJFFixbB0dERs2fPRkhISJXzQkRERNbNoCDIx8cHMTExGDNmTIlmJrVajZiYGINGaI0dOxYbN27E1KlTMXnyZCQlJWHJkiUYO3asXh+jCRMm4M6dO9i7dy8AIDQ0tMS5FAoFnJ2d0alTpyrng4iIiKyfQUHQCy+8gLfeegujR4/Gk08+CX9/fwBAQkICNm/ejIsXL+Ktt96q8nnd3d2xYcMGLFy4EFOnToWLiwtGjhyJWbNm6aXTarXQaDSGZJ2IiIgIACARDBy2tW3bNnzyySdISUkRZ48WBAFeXl6YOXMmRo8ebdSMmoJGo0VqapbRzyuXS+Hp6YK0tCyrbOtl+SyftZeR5bN81l5Gls9wXl4uNdsxGgBGjRqF4cOH49y5c+ISFQ0aNEBYWJhBI7GIiIiIalO1ohW5XI6IiAhEREQYKTtEREREtcOgqRovXryIX375RW/fgQMH8PTTT2PUqFEVzgxNREREZGoGBUEffPABYmJixO1bt25h2rRpSExMBAAsXrwYW7ZsMU4OiYiIiGqAQUHQpUuX9GaD3rlzJ6RSKXbs2IFt27ahX79+2Lx5s9EySURERGRsBgVBGRkZ8PDwELf37duHqKgoeHl5AQCioqKQkJBglAwSERER1QSDgiBvb2/ExcUBAJKTk3H+/Hm9GZ2zsrIglVrnyrdERERkHQwaHdarVy988803yM/Px+nTp2Fvb48+ffqIr1++fFlcWZ6IiIjIHBkUBM2cOROpqanYuXMn3Nzc8N5776Fu3boAgMzMTMTGxuLpp582akaJiIiIjMmgIMjFxQUffvhhqa85Oztj//79cHR0rFbGiIiIiGqS0ad2lkqlcHNzM/ZpiYiIiIyKvZeJiIjIJjEIIiIiIpvEIIiIiIhsEoMgIiIiskkMgoiIiMgmMQgiIiIim8QgiIiIiGwSgyAiIiKySQyCiIiIyCYxCCIiIiKbxCCIiIiIbBKDICIiIrJJDIKIiIjIJjEIIiIiIpvEIIiIiIhsEoMgIiIiskkMgoiIiMgmMQgiIiIim8QgiIiIiGwSgyAiIiKySQyCiIiIyCYxCCIiIiKbxCCIiIiIbBKDICIiIrJJDIKIiIjIJjEIIiIiIpvEIIiIiIhsEoMgIiIiskkMgoiIiMgmMQgiIiIim8QgiIiIiGwSgyAiIiKySQyCiMgsnbr6AJ9sO43LN9NMnRUislIMgojI7Kg1Wqz95QLOxKVgxQ9nkZ2rNnWWiMgKMQgiIrNz414GsvN0gU9Wrhp7jt00cY6IyBoxCCIis3PxRqre9p5jt5CZU2Ci3BCRtWIQRERm52KCfj+g3HwNYv5JMFFuiMhaMQgiIrOSX6DBtdsqAIDCxR5yme4x9cfxRKRn5pkya0RkZRgEEZFZuXZbCbVGCwBoHVQHPds2BADkq7X45dANE+aMiKwNgyAiMivFm8JCG3tiYOfGcLCTAQD2nbqDB+k5psoaEVkZBkFEZFYu3NAPghQu9ujTwQ8AoNEK2HnwuqmyRkRWhkEQEZmN7Fw1btzT9QdqUNcF7q4OAID+Hf3h7CAHABw6dw93U7JMlkcish4MgojIbFy+lQZB0P07tLGnuN/Z0Q4DOvsDAAQB+PEAa4OIqPoYBFmJArUGH2w6if9bewSpqlxTZ4fIIMX7A7UoFgQBQO92jaBwtgMAHLuUjJtJGbWaNyKyPgyCrMSJKw9wMSENtx9kYe+/t0ydHSKDFAVBEgkQ4u+h95qDvQyDIgPE7R3742sxZ0RkjcwuCIqLi8Nzzz2HiIgIREVFYcmSJcjPzy/3mOTkZCxZsgRDhw5FmzZtEB0djVdffRW3b9+upVyb3vW7KvHfj040R2QJlFn5uH1f19cnoJ4bnB3tSqTp3qYBPN10/YROx6Ug7rayVvNIRNbFrIIgpVKJCRMmoKCgAMuXL8esWbOwdetWLF68uNzjzp8/j71792LAgAFYuXIl5s2bhytXrmDUqFFITU0t91hrEV8sCLqVlMklBsjiXCoWvDd/pCmsiJ1chiFRAeL2D6wNIqJqkJs6A8Vt3rwZWVlZWLFiBTw8PAAAGo0GCxYswOTJk+Hr61vqce3atcOuXbsglz8sTtu2bdG9e3f8+OOPeP7552sj+yaj0Wpx897D/hECdF8o7Zv7mC5TRFWk3x/Iq8x0UeH1seufm0hOz8HFhDRcuJGKFgFlpyciKotZ1QTt378fkZGRYgAEAAMGDIBWq8XBgwfLPE6hUOgFQABQr149eHl5ITk5uaayazbuPMhGvlqrt49NYmRpLiboam3lMgma+rmXmU4uk2Jotybi9g/74yEUDSkjIqoCswqC4uPjERgYqLdPoVDA29sb8fFVq/a+fv06UlJSEBQUZMwsmqXi/YGKMAgiS/IgPQf303WjGoMauIszRJelU6gvGtZ1AQDE31Hh9LWUGs8jEVkfs2oOU6lUUCgUJfa7u7tDqax8B0hBEPDuu+/Cx8cHgwYNqlae5HLjx4mywgUhi/5fXQnFmsIc7WXIzdfgXmo2VNn58FI4GuVvVIWxy2durL18QO2X8XJiuvjvlk28KnXfPdE9CMu2nwEA7DgQj7bNvSGVSCr196z9Glp7+QDrLyPLVzvMKggyluXLl+Off/7B2rVr4ezsbPB5pFIJPD1djJgzfQqFk1HOk5CcCQCQSoD+kQH4cV+cbv/9LAQ1rmOUv2EIY5XPXFl7+YDaK+O12w8D+U6tGlTqvuvdOQAxR27i2q103ErOxIUEJbq1aVilv2vt19DaywdYfxlZvpplVkGQQqFARkbJCdCUSiXc3cvuI1Dc1q1b8dlnn+F///sfIiMjq5UfrVaASpVdrXOURiaTQqFwgkqVA41GW/EB5chXa5BQ2BxWv64LWvh74MfC146dv4c2QbUfBBmzfObI2ssH1G4ZBUHAqau6vnsOdjJ4u9kjLa1yy2IM79oEH2w6CQD4OuYCQv0VkEkr/mVp7dfQ2ssHWH8ZWT7DKRROla5hMqsgKDAwsETfn4yMDNy/f79EX6HS7N27F2+//TamT5+OkSNHGiVPanXNffg0Gm21z3/9tgoara5TaEA9NzT2dYO9XIp8tRbnr6eioEADSSWbCIzNGOUzZzVRvpw8NRzsZJBKTXPNHlUb1/D2gywoM3VzgQU38gCEyt93zf09ENzIA1dupeNeajb2n7qDbq0aVPpvW9JnVK3RQqsVYF9Bf6niTF0+tUYLiQSVCkwNZeoy1hSNVot7KVlQKrOh1lTc8d/RXgaFs30t5My4TH39zCoIio6OxqpVq/T6BsXGxkIqlSIqKqrcY48cOYLZs2dj1KhRmDp1am1k1ywUnx+oSX0F7ORSNPNzx/kbaUjLyENyWg58vQxvEqTa8+fJ29j021V4ezjijXHt4exoVrdnjSk+P1BoGfMDlUUikWBEdCAWf3sCAPDT3zfQuUU92NVAXz5TysjOx9tfHUNWbgFee7odGtdzM3WWKpSdW4Al353E7QdZmDM2AiH+Vbu2tuz6XRU+23EWqaq8Kh03qkcQBnRqXEO5sk5m9aQYO3YsXFxcMHXqVPz999/4/vvvsWTJEowdO1ZvjqAJEyagT58+4nZcXBymTp2KgIAADB06FKdOnRL/u3nzpimKUmtuPBIEAUBosTlTLnCUmEXYffQmNu6+DLVGi7sp2dh91Lo/t8VduPFwQtOqBkGArvYoLFD3mU9R5WL/6TtGy5u5+PdSMtIy8pBfoLWYz0bs0Vu4mZwJjVbATwdvmDo7FuNqYjqWbj5Z5QAIAHYeuI70zKofZ8vM6qemu7s7NmzYgIULF2Lq1KlwcXHByJEjMWvWLL10Wq0WGo1G3D59+jQyMjKQkZGBJ598Ui/t8OHDK5xx2pJdv6vrQyWXSdDIxxWA/hfJxYQ09KhiZ1GqPYIg4JdDN7DjkVXR9/x7C73a+1lk9XZVaLUCLt9MBwC4OMrRyNfVoPOMiA7EuXhdMPXLoRvo2qp+hcPsLUnxHzMnrz5AXoHGrMunysrH3mMP1zC8dDMNysw8uLs6mDBX5u/ijVR8+v0Z5BfomoeaNFCgnqdzhfNgJaXl4PpdFfLVWvxy6Aae6RtSG9m1CmYVBAFAUFAQ1q9fX26ajRs36m2PGDECI0aMqMFcmafsXDXupeo6bjfycYW8sCNYY183ODvIkZ2nxqWENGgFodJDh6n2CIKAH/bH49fDCeK++nWccTclG3n5GsQcTsDYXs1MmMOal5CUgew8NQDdUhmGfk4D6inQLtgbx6/chzIrH38cT8SAztbRLKAVBL0mw7wCDU5fe4COoaXPoG8OYv5JQF7Bwx+qggAcu5SM3u0bmTBX5u1M3AOs+OEc1IWdhMMCvfDWi5HIycqrsM+MKisfc1cdRl6BBvtO3UH/jv6o62Hdo8qMxayaw6hqbtx72BQWUP/h/EpSqURcgTszpwCJhUPoyXwIgoBNv1/VC4BG92iKOWPbiP1Z/jhxG2kZ1l21rb9URvX6jAzr1gRFIVTMPwnIzlVX63zm4lZSJrIeKcuRC0kmyk3FUlW5+OOEbvFqWbEO/kcumm+eTe345ftY/v1ZMQCKaFoXs0ZHwNG+cvUUChd79G7vBwBsfqwiBkEWrPhM0YH19SeZfLRJjMyHVhDw9e7L+O3fRHHfM32D0b+TPzzdHNCrre5hptZo8fOhGybKZe24WIlFUyurobcrOresBwDIylVjzzHL6DtTkdLu37PxKcjONc9Fkn85dEP8Mu/ToRH8vHVzPsXdVuFBeo4ps2aW/rlwD5//eE4c5duhuQ9eHh5W5c79/Tv5w8lBFzQdPHcXd1MqN82ErWMQZMFu3H04p1LAo0FQsc7RDILMh0arxbpfLmLfKV3nXYkEeH5gKHoWBj4AMKCzPxztdf09Dpy+g2Qr/eIoUGtx9VY6AMDTzQH1jDCKcWi3JmLtw55jt5CZY56BQlUUv39bBugCRbVGwPEr902VpTIlp+fgwJm7AHRDtgd2boxOLR4227E2SN+B03ew5qcL0Bb2+YkKq4fJQ1qKXRuqwsXRDv07+QPQNT/u/Pt6BUcQwCDIol0vbA5zsJeh/iNfIA3qOMPdRdep9vKtdPGXGZmOWqPF6p3ncfj8PQCAVCLBpMdbomur+nrp3Jzt0beDru+ERivgJyt9mMXfUYoL/4Y29jTKfFY+Hk7oVvh+5uZrEPNPQgVHmDe1RosrhYGiu6s9hnV7OF/a0Yvmtzj0zgPXxRqNvh0awdXJDh2K9V0yxzybyu/HE/HVrkso6vLcvU1DPDcotFpzhPVp7wc3ZzsAuvf6ZlLJyYdJH4MgC6XMzBOHUAb4upW4cSQSidgklpev0as1otpXoNZg5Y5z+Pey7te7TCrBy8PD9H4lF9e3gz9cCucJOnz+Hm4/sL6q7YvVmB+oPIO7BIi/pP84nmjRQ4av31WJHYxDG3sisIECdd116wFevJEGVVa+KbOn5/aDLPxTGOC7OMrRr6OuVsLHwwmBDXQ11beSM3HHCj/LVRV75Ca+3XtF3O7TvhHG9Q2u9gAWR3s5BhUbEPDjAev8AWVMDIIs1PViQU2TR5rCijTX6xeUWmoaqnl5BRos234Gp649AADYyaWYPrIV2gZ7l3mMs6NcHN0kCMCPB+LLTGupaioI8lI4omdb3bQQRUOGLdXFG8XeI39dbVlR4KwVBBy7ZD41Kz8eiBdrNQZ2biz2TwGATsVqg8y5U3dNEwRdze7WP6+J+wZ3aYyxvZoabWb/Hm0bwtNNNxXBqWsPEHe78ouP2yIGQRaqeKfoJg1KD4JasHO0yeXkqfHx1tM4X/hl5mAnw8xRrREeWPGabr3a+kFR2KR5/PJ9JNyzntq83Hw14u/oPsO+nk7wUjga9fwDOzcW59HZd+pOtTrkZucW4PpdVYVztdQEvUCxsD+QXkBhJn1sbtxT4XhhLae7iz16tvPTe71DqI84cu/IxSSTvJeVlZVbgMT7xh9RKwgCtu+Lw4/FmreHRwdiRHSQUZc2spPL8HhUgLj9w37r+wFlTAyCLNT1YsPjm5QxhX5dDyd4e+i+XK7dVurN20G1Y+Puy2KfDicHGV4dE1HpWg8HexkGRz6s2ramh9nVRKXYd6R4J35jUbjYo0+Hh0OGdx40rFkg4V4G5q3+Bws3/ItfDtdu/6K8Ag3i7uh+xXt7OKKuu27el4beLmhQVzfi6lqiEinK3FrNV2mKfzYHdwkoMZGjh6uDOG1HcloOEsy0r0pegQbvfn0c89cdxS4j9yf78+Rt7Prn4YjFsT2b4vEuAUb9G0W6hteHT+E8QRcT0nDxBlsCysIgyAIJgoDrhb+iXZ3sUMe97F/RRV+4ao2Aa6wWrVUFao04gsfJQY7/PNkGTf3cq3SOxyIaoo5CV7V9Nj4FVxPTjZ1Nk9Br5jFiU1hx/Tv6w7mwSebQuXtVHjJ87bYSSzadFEeY/Xr4BpS12L/oamK6uHBmaOOHgaJEIkGnUB9x++gl09YGXbmVLs7WXUfhiMciSl/AVm+UmJk2iZ2+9gBJhRPQ7jx43WjXOzdfrTdaa1y/EPQt7DNVE+QyKYZ2bSJu/7A/3qxr30yJQZAFuq/MFSdPa1JfUW5VavGHZ/EvHqp51xKVKCgc/dQuxBsB9UpvtiyPnVyKIVHFHmb7rONhdqFYH7XmhTUExubsaIcBnR8OGa5KJ9FLCWn4cPMp5OQ9nKQwv0CrN7llTSuvz1RHMwkoimY9LzKka0CZw7vbhfiI0xccvZgsDgs3J8XfS2Ne773/JiIjWxdMdwz1qZWljDq18BVrDOPuqHA6LqXG/6YlYhBkgfQXTS1/Nenm7BdkMhdvGmc25C7h9eBbOAXC5VvpuGDhwWxmTgFuJen6XDTycYVbDa6P1rtdIygKhwwfu1S5IcPn4lPw8bbTYvNxSCMP2NvpHpV/nbpda81P5dWW+Xo6i/f+zaRMk02Md/5GqtjcW8/LGV3C6pWZ1tXJDmFNdD/K0jLyxDmizEV2bgHOxusHCsa43lm5BYg9omsGk0oketMc1CSpVILh3R7+gNqxP94sA09TYxBkga6XsnJ8Wdxd7NGwcMbWG/dUZjvLrDUq/iVWndmQZVIphhWr2v5+X5xF1wZdSkgTRxHVVFNYEQd7GQZFBojbOyroV3Xyyn0s+/6MWIPXKqgOZo9pjT6Fa16pNQJ+MrB/UVVk5RaI/Wb8vF3EDvLFdTTxiCtBEPDDvofv57BuTSCTlv+VUrwGy9zmDDp+5b7Y/FjUjGqM6x175KZYo9glvJ5RJgWtrLbB3mhc2Gf0VnIm/jWj0YTmgkGQBarM8PjiQv11XzSCoKtJoJqXk6cWr1ODui7wqObq2R1CfeDnrVth/ca9DJy8+qDaeTQVvRqygJoNggCge5sG4pDh03EpZQ4ZPnoxCSt/PCd+EbYL8ca0EeGwk8v0lyQ4e0/sN1JTLt9MR1GcW1YA3THUVxxxdfRicq0HxievPsCNwhGLjXxc0b65TwVHAG2a1YV94XIQxy4lm9UkrkeLBZKTh7aEk4Ouc3d1rrcyKx97/70FQDc32JBio7Zqg0QiwYjohzVPPx64Do3WfN5zc8AgyMJotYI4VLqOwqHUX4iPCi32RcN+QbXj8q10seq5KAitDqlEguHRxaq2D8RDq7XM2qCiz6BMKkEzP48a/3t2cpnel09po+wOnr2L1T+dF0esdW7piylDHy5f4OJoh/4ddbVBWkHQG+ZcE/QXli199JynmwOCG3kAAO6lZuNmUu0tlKzVCthRbO6q4d0CKzXRn6O9HK2b1gWgaxY1lyZ6ZVY+LhTmxdvDEWFNvMTJHqtzvX89fAP5Bbqgo3tEQ3GEX20Ka+KF4MIBGfdSs3H4nHl2SjcVBkEW5k5KlthXoTK1QICuT0PR88lcHjrW7lIp87tUV0TTuuI1v30/C0fNZI6YqkjLyMO9wl/VTeor9CbUq0lRjwwZPn/9YcfsP0/exrpfL4o1L9Gt6+OFQS1KNO30bq9bBgLQ1RokJtdc0FF0n0okEAOd0phqXa6jF5Nw+76uH1JQAwVaN6143qsi5jhK7N9LyeL17xjqC4lEgj7VvN6pqlz8dfI2AMBeLsXgLo0rOKJmSCQSjHgsSNz+6eB1s6qBMzUGQRamKv2Bijg72okjk24/yILSjKbat1ZFnZclEojzo1SXRCLBE48Vq9r+2/IeZsVnLq/uqvFVIZdJMaxYJ9Htf12DIAiIPZKAjbsvi/t7tfPD+P7NS12/yclBjkGF8zYJgF5NiDEpM/PEpSWa1FfA2bHsQLFdiHexEVdJtdLxVa3R6tWMjIgOrNJkf+GBXmJT04kr91GgNv38ZcUDyKLJKKt7vX86eENsWu3V3g/u1WwSr47gRh5ip/QHylzsP33HZHkxNwyCLEx5K8eXp3jfCy6hUbNUWfnijLONfd3g4mhntHO3CPASh5Qnp+Xg0Ll7Rjt3bdBv5qm9IAjQdcotGiQQd1uF9zYcw3d7r4qvD+jsj6d6Nyu3WadHm4bwcNU1QZ+8+kCc9dqYqrKciJuzPVoUTjaZqsqrlSUSDp69i+S0HDF/VZ3s0k4uE5eMyc3X4IyJh26nKHNxLVH3vjWs6wI/H1fxtUevd9HklRVJSsvG32fuAtBNkjqgk2lqgYobXqxv0M+HbnDy3EK1UxdNoqzcApy/kYaubfwqTlyK+MKaIAmAgDJmii5N88ae4pwXlxLS0LlF2UNZa9Ot5Ewkp+WgddM6Zc4vYogCtQbHL99Hembla738vF0QVonlLCpy6abxm8KKG/FYEBZtPA5AV7Ud2dIXdnJZBUeZniAI4he8nVyKoIZVnzepOqQSCYZ3C8SKH84CAA6fvSu+NqxrEzweFVBhjYa9nQyPRzURa4927I/Dq2PbGDWfF6q4plqnFj7i0O4jF5Kq3M8qN1+Nfy/dFyeFrEhRR18Aep1uq6JTC18cPKsL4I9cSEK7kIo7VdeU4s3KHR9Z0Ljk9Y7HnEpc751/Xxdr5fp18Beb1UypSX0F2gZ748SV+1Bm5uPPE7fRv5NxJ2y8lZyp19RcHqlUgrpezmjZ2AMOJnx+MQiqZRt3X8bRi8n49fANLHi+Y5WOLVBrxXbpenWcq9SfollDd8hlEqg1gtnMM3Pg9B2s33UJAnRDkacODzPKl3l2rhqfbDtt0AzZs0e3rnYgVFMLgxZp2tAdrYLq4ExcClJVeVi/6xKeHxRa4fBkUztw5i5SVboZeJv5uZskcGvTrC6a1HfTG2E5ukfTKn0ZdGtVH7v+ScADZS7O30jDpYQ0ozbtFfUnk8ukaNqw4hnG2zTzhp38MgrUWvx7KRlP9m4GeSUr+ZWZeVi6+RRuG7Cye+ugOgiqRP5KE9rYE27OdsjILsDpuBTk5KlrrX/Yo4r3Syo+E3eR4tf7QiWud+L9TBw5rzunq5Md+nRoZPxMG2h4tyY4eeU+BOgCtSb13RBihIEbgP7zvCo6hvpgytAwo+TBEOb91LRit5Iysf9U1dplbyVniqNXKtsfqIi9nUx8oD5Q5uJ+NRaUNIbfjyfiq2I3zJm4FHy6/Qzy8qtXRZuZU4APNp80eImQ/WfuVpyoAkVBUE2OfhoRHSj2BTl8Pgmrd5436/5Bvx9PxPpdl8Tt4nPc1CaJRILRPZpCKpFAIgHG9w+p8q/hmlySIDk9Bw8KJ+dr2lABe7uKA0UnBzlaBekCd1V25UdcpapysfjbEwYFQHZyqV5n26qSSaXikPoCtRYnr943+FzVcTclCzcLf1g2qe8GH8+Sc/hU9Xr/eOC6+Fwb2LmxyYK70jT0dkVUeH0AunXSPt56utI1N+V59HleFc5G7C5gCPO5Ojaibwd/cZKwnX9fR+cWvpV60AGGdYouLrSxJy7dTAeg+6L29qj94ZoAsOtIArb9GSduy6QSaLS6GqqPtp7CzFGtDXpwKLPy8eHmk0gsHLXi6mSHsb2awr4SNQ5f776MzJwCnL72oFq/SlOUuWJ/iaCG7iUWkjQWf183vDQsDJ//eA4arYB/L9+Hesc5vDSspdk1jT16vft2aIRureqbLD8h/p5YNLkzFAonuNpLoVZXPXiMbFkPMf8k4G5KNq7dVuJsfApaBdWtdt70RxVWvq9Np1BfcRX3oxeSEdHMu9z0yek5WLrppBhw1VE4YGT3pmJgXZFGvq7wLSVgqIpOob7484Ru9NTRi8noElb7nwn9WqCyA/PKXu/rd1U4UbheoLurPXq2rfnlMarqmb7BUGbl42x8CvLVWny6/TReHhaOiGaGfX4fvb+7t2lYqf5+MpkEvnXd4FfHyaTTfTAIqmWBDRRoE1wXJ688QFpGHv48eVucj6IixYOggAqWyyhNaGMv7ChcP+liQhqiW5e+0GFNEQQBPx28obeQ4OAujdE6qC4+2noaOXlqXE1UYunmk5g1OqJK7ehpGXn4YNNJcfi1u6s95oxtg4aFa+dU5GJCGv48eRsFai1OXX2AyHKm/y9P8TWxarrjb9tgb0wf2Qorfjiry/e1B1i2/QymPdGqxoKvqij9egdgeLcmVRpNVBMa1HWBp6cL0tIMW25CtyRBIFb+eA6ArnYgLLBOpebKKc+FYqt9V6UptVVQHTjay5Cbr1u0t6CcwO5uShaWbj6FtAxd06SPpxP+M7ZNuQsx14Smfu7wUjggVZWH89dTkZlTUKt9ZwRBwJHCH6QSAB3KCYIqe72Lz0H1eJeASv/ArU32djJMGxGO1T+dx4nCWbI/23EWk4a0RIdKTHhZpKzn+fBulRstKJdLxXvQlEEQm8NM4InHgsR5e349nKC3SGN5ioIgmVQC/2IjGCoroL4bHOx1N+XFhLRanWFWEARs3xend8OMiA7EiOggBDV0x3+fbCM+AK/fzcAHm05ClV25Ts0P0nOw+NvjYgBUR+GAeU+3rXQABBhvvpXiv+RrYwh4eGAdzBzVWgx6zt9Iw8dbTlX6M1VTBEHA9r9Ku95VG05tztqFeKOx78P1u05crl6TjiAI4ufH0V5W4bqAxdnbydCmsPYnJ0+NM3GlzyiemJyJ9789IQZADeq6YN7TbWs9AAJ0HdU7NtfddxqtUOtLOtxMyhRngg7x9xBnFS9L2xBv+Pu6isc+er0v33w4/1Rdd8da/5FZFXZyKV4a1hKdWzx8/1ftPIdD5yrXHaC05/nwwue5pd3fDIJMwN/XDd0idNWkmTkF+K3YaIuy5OSpcS9Fd8P6+bga1OQhl0kRUjjxmiorX5yLpKZpBQHf/XYVu/65Ke4b26sZBncJELcb13PDf59qI86AfavwYZ2emVfuuZNSs/HetydwP11Xre/t4Yi5T7etclV9Uz938SFY9Ku0qgRBEEf2ONjJENigdkY/hTb2xKtjIsS5V64kKvHhllPIMtE6cVpBwHd7r2LXkbKvtzWQSCR6w46rO4v37QdZUBWuNB7cyKPKHd2LB/KHz5cM5G/cU+H9706If6ORjyv++1Sbai/pUh2mnDix+N+rTB81qUSCEdEP+0EVv96CIOjVAg3t2sSoo11rgkwqxQuDW6BrYdO0IADrfrmIv07dLve4Up/nPZvicQu9v837Klmxp/s1F6tSY4/eqvBLN+FehtjpzJD+QEVCa3lVea1WwNexl/D78URx3/h+IehbyogJP29XzHu6rRiM3E3JxuJvT5S5ivPt+5lYXOxXbf06zpj3dDuDpqaXSiRinwBdH5uq/yq9l5oNZeGQ/GaN3Gv1IdjUzx3/ebINXAon1ou/o8IH31W+Ns1YtFoBG3Zdwu8nKr7e1iA80AtNC5ckuJuSjcPnDZ+3qfiSNoY0pbYI8BRrU09dua9XG3gtUYkPNp1EVq5uX5P6Ct2PDueKl92pSf6+rvAtXFD0yq108V6uaVpBwNFLuiBIJpVUat0zoOzrfe56Kq4WzjVUv44zIluaxxQkFZFKJXh2QHOx75IA4OvYy9h7rPQf5qU9z8f1C0HfSnbpMEcMgkykgbcrurXWReA5eWrEFvvVXBq9TtFVmB/oUbUZBGm0Wqz99QL2n9ZVsUokwMRBoejepuzOgvW8nDHv6baoW1g9n5yma+pKTtNfwDDhXgbe/+6kOPu1n7cr5j7VtsIq7fJ0bPHwQXjUgF+lF/S+xKo2gZwxBNRTYO5TbaFw1n0R3kzOxJLvTlZYm2YsGq0Wa3+5gANnKn+9LZ1EIsETxWqDdlZjFu+L1WxKlcsejrjKV2txpPAL+lJCGj7ccgo5ebqRl8F+7pgzNsKok3gaSiKRiMPSBQDHamnpj2uJSnG6hpZNvCrdF6m0612g1urVAg3rFljqjOPmSiqR4Ok+wXqjJDf9fhW/Hr6hl66s53kPC7+/GQSZ0NBugZDLdDfLb8dvQVnOl5VeEFSNZhY/H1fxhr90M73GOqQVqLVY+cM5/HP+4a+tyUNaisMzy+Pt4YR5T7cVfyGmqPLw3rcnxOa7uDu6X7VFtWcBjzSlGaqxrxt8PXW1SJdvVv1X6aUanh+oMvx8XDG3WG3anQdZ5damGYtao8WqH8/jnwsPr/eUoWGVut6WLsTfEy0LJ8V8oMwVg8Cq0Gi1uHxL9/lxdbLTm7W4KorPc7P/ZCLOxD3Ax9tOi7MDtwjwxKzREWY1bNsU65+VtkxGZT16vVfuOCsuau3v44p2IeWPzDNHEokEo7oH6S00/P2+eHE6ALVGi1U7zxv0PDd3DIJMqK67Ix4r7BuUX6AVZ3QuTdHkbg52MjSoU/kOv4+SSiTir8ycPDUSkjIqOKLq8tUaLFp/FMcKOzrKZRK8PDysSnPDeCkcdZ2bC5c5UGbm4/3vTuCvU7exdPMpZBdW9Tf1c8ecsW2MMqpEIpGID2QBEPNfGVqtIM4U7eIoRyNfw77EjKF+HRfMLVGbdqJEbZqx5BdosOKHszheODS46HpXZaSJpRterK/IzwevI7+KSxIk3MsUa2qaN/Y0eJRZs0YPO/ieuJSMT7aeFkeKtQqqgxkjW4mDI8xF/Tou4kCP63czkFRDn9MiGq1W7IRtJ5caNDS8+PU+XWzZj+HRgdUeIWgqEokEw7oFYmT3h2X75dANbPnjmu7+vqx/f5tqri9jM5+fAzZqcGRjHDh9B/lqLf46pRsu/+hIDVVWPlJUul/yjX1dq13VGtrYU3wIbPr9KupVc76PR91+kCXWXNnLpZj2RDjCmlR9FmZ3F3v898k2+GjLaSQkZSAjuwBfxz5c7DK0sSdeeSIcjvbG+xh3auGLnw7eAKDrOFnZviw3kzPE/hbV+RIzFp/C2rQlm04iOS0HKYUT4/3nyTaoX40g+lF5+Ros+/6M2JRTnettyQIbKNCmWV2cvPoA6Zn5+KOKSxJcNNLUClKJBB2a+2DPsVvixKqAbiTb5CEtzbazbqcWvuKkhV/+erFSAxtkMgm6t/dHUL2q/eC4mJCGjMLO4a2D6hhUK1b8ehcJaqgQJ620ZAM7N4a9XIrvftOtq7enWP8ga7y/GQSZmLurA3q198Ouf25CrRHw86HreHZAqF4a/fmBqj/iqPhD9lqiUlw80Ngc7WWYMbJVtaZld3O2x3+ejMDH204j7vbD96FVUB28PCzM6PNwFP0qvZmciet3VUhOyy51FtlH1fRSGYYoqk1buvkU7jzIQnpmPhZ/ewJzxrZBIwObW4rLyVPj422nxc+Pg70MM6t5vS3Z8G6BOHX1AQTo5oupV8cZEU0rV8tgzM9Ppxa+el9ckS19zX5ZlQ6hPtj2l27CvauJSrGTcUX2nbqDsb2aVanjvd4EiS0Mr80ofr0BWOTw8LL0bt8I9nYybCg2C7S13t/me1fYkAGdGovDm/8+c0+cu6JI8SDIGMOufTydEG6EhULL4+nmgP8+1dYoN4yzox1eHRMh5rlzC19MGxFeYxORddTro1C5JrHiI3vMJQgCAA9XB8x9qo04v0lGdgGWfHdC7zNliMycAizdfFIMgJwc5JgzJsLqHpBV4efjiugI3dwwao0Wn/1wtlJNqgVqjfil76VwgI9n9WZyD6jnJt4rvdr5YeLgFmYdAAFAXXcng0dUbf79Kn76+3ql5j0rUGvEGZ0d7WXVeg76+biKE912CatnVve9MUS3boAXHm8BB3sZ3F3srfb+Zk2QGXB1skO/jv748YBu5eEf/76OyUNaiq/fuPew344xaoIkEglmjmqF5PQcaDTG7xgtk0vRPLAuMlQ5Bi1JUBpHezlmjW6NrNyCGh/V0jHUB9sLf5UevZhU4fwXao0WVxLTAQAervao52Xc5sXq0tWmtcHHW08j/o4KWblq3azcoyLE4b5VocrKx9LNp5B4X9d84eqkC1IbV2PUorV4uk8wcvM1OHIhSZyATq1uUe4M5HG3VWK/ndDGntWuTZBIJJjzZATkDvbQ5BcY7R6saS8MDsWwbk3Kne26uONX7mNH4aisH/++jny1Fk88Vv5knGfiUsW+V22Dvav9Q2pUjyAMjGwMZ0fr/CqNbFkPbZrVhVwmNdum1Oqyzitngfq0b4Tf/k1EZk4Bjl5IwqDOjeHn4wpBEMRf7a5OdvA20syuEomk2mv/lEUur7kbpjaG9dZ1d0LThu64dluJ2/ezkHg/E37eZTcfxd9RIb+g6EvMyyyrxF0Ka9M+3X4GV26lIydPgw+3nML0J8KrtEZVWkYelm4+ibuFE3cqXOwxZ2xEue+PLZHLpHhxcAvYyaT4++xdCAKw9pcLyFdrxEEQj7pQA02pEokEChd7pOWbZsJMQ0gkkiqtZ+hfzw1eHk5Y99N5AEDMPwnIL9BgbO9mZfbJO3rROE1hRSQSSa0u9WEKxuxzaY6sM7SzQE4Ocgzs3BiAbmTSjgO6XzgpylyxE19APTez/IK1RlWZydbQ9Z5qm5ODrjataHhvXoEGn2w/gzPFRreU54EyB+9/e0IMgDzddMuTMADSJ5VK8OxA/QnoNpQzAV3xTtGhJphfypINe6wpJgxoLm7/djwRX8deKnXqj9x8NU5f03VkdnWyM+t7lWoPgyAz0rNtQ3i46ua6OXn1AeLvqHC9WFNYdWaKpqpp39xHXN/t6MWkcvsbmMP8QJXlYCfD9JGt0LpwFEuBWovl358Rh7+WJSktG+9/ewLJ6TkAdNM7zHu6rdk1/ZkLcQK6juVPQJeTp8b1O7p7vJ6Xc7Um+7RVvdr5YeKgUPF+3X/6Ltb9egEarX6z2smrD5Bf2NTWvrmP1TbvUNXwU2BG7O1kev1PduyP058kkUFQrXF3sRcDmvvpueI8TY/Ky9cg7o7uGvl4OplkIcqqspPLMHVEuDi7sEYr4PMfz5VZ4yVOuFg4w65v4azeVWm6sEUSiQSjepScgG5H4QR0gG6pCG3hv0MDzDuANmdR4fUxeUhLsRns8PkkrNp5Xm/2br1RYaG2M4cVlY9BkJnp1rqBOMnd+RtpOHTu4VpEVVlVmqqv+EyyZQUIVxPTxflYqjO/S22Ty6SYPKSFOCJHKwj44qfzOHDmjl66m0kZeP+7E+KaaA29dauOeynMP9gzB0UT0D3x2MOlFn4+dANb/7wGQRD0h8Zb4cib2tQx1BdTh4eJs/Afv3wfn/1wFgVqDTJzCsQV3j3dHNCscCFpIgZBZkYuk2Jo1ybitqpwbSxPNwe4m3C1Z1vULsQbssKJKY9eSiq1n4Fep9YqdDA2BzKpFBMHh+KxwmHdAoCvYi7hj8LFT6/cTMN7G4+LfdIa+7rhv0+2gXs1lyexRYMiA/Bk72bi9u6jt/DNnivienMSGLZeGOlrE+yN6U+0gp1c99V2Oi4Fn24/g8Pn7ok/VjqG+ph8MlMyHwyCzFBky3qoX0e/r0Ugm8JqnbOjnTiPiDIzH5dvpZdIU/yXfIi/Ry3lzHikEgnG9wtB73Z+4r5v9lzB17GX8OaqQ+Is2EENFfjPkxFwM/Gq45asT/tGmNA/BEVfv3+evC1OM9DI19XqRxnVlrDAOpg1qjUcCoe/X7iRhs2/XxVft5blHsg4GASZIalUguHdAvX2BbApzCSKjxI7+sjijpk5BbhZ2HG9kY8rFBYaIEgkEjzZu5k4OhEAfvs3ETmF67M19/fAq2Mi4GwGq45busciGuKFwS3waEVEC44KM6rmjT3x6tgIcRLaojpcH08nBHA+KyqGQZCZahviLc7yC7BTtKlENK0LezvdbfLvpWS9jpYXE9LEh6u5jwqriEQiwROPBWJYtyZ6+8MD62DGqNZWP1dIbYoMq4eXhoaJTa0Am8JqQtOG7vjPk23gUmwiw06hvpxmhPQwCDJTUokEE/o3h6ebA8KaeFlkU4s1cLCXoU0zbwBAVq5a7FwJABctZH6gypJIJBgS1QRP9wmGp5sD+nT0x8zRD5sVyHjaN/fBtBHhqOvuiIimddGyieV/fsxRQD0F5j7VFn7eujUBexVr9iUCOGO0WWtSX4EPp0aZOhs2r2Oojzg67MjFJLQrHFpeFBBJJRIEW9Fok17t/NCvkz88PV2QlpZlMcsuWJrWTeuidSUXWCXD+fm44p2JnUydDTJTrAkiqkBYkzpwdtD9Xjh59QHyCjRIUeaIMyc3aeAGJwf+niAisjQMgogqYCeXol2IrkksL1+D01cf4Ezh9PuAdTSFERHZIgZBRJVQfJTY4fP3cPrqw2UmuN4TEZFlYh0+USU09/eEwsUeqqx8nLmWAldn3XBxO7kUTRty5B4RkSViTRBRJUilEnQo7BBdoNEiLUO3jlbThu6wk3P0FBGRJWIQRFRJxZvEirTgopdERBaLQRBRJQU1UKDOIwuHcpI7IiLLxSCIqJIkEgk6tvARt50cZJyCn4jIgjEIIqqCTsUWXwxt7AWZlLcQEZGl4ugwoirw93XDyO5BuJKoxMjuQabODhERVQODIKIqGtK1CZeUICKyAqzLJyIiIpvEIIiIiIhsEoMgIiIisklmFwTFxcXhueeeQ0REBKKiorBkyRLk5+dXeJwgCPjiiy/QvXt3tGrVCmPGjMGpU6dqPsNERERkkcwqCFIqlZgwYQIKCgqwfPlyzJo1C1u3bsXixYsrPHbNmjVYtmwZnn32WaxevRre3t54/vnncevWrVrIOREREVkasxodtnnzZmRlZWHFihXw8PAAAGg0GixYsACTJ0+Gr2/JZQsAIC8vD6tXr8bzzz+PZ599FgDQrl079O/fH+vWrcPbb79dOwUgIiIii2FWNUH79+9HZGSkGAABwIABA6DVanHw4MEyjztx4gQyMzMxYMAAcZ+9vT369OmD/fv312SWiYiIyEKZVU1QfHw8nnjiCb19CoUC3t7eiI+PL/c4AAgMDNTbHxQUhA0bNiA3NxeOjo6lHVohudz4caJMJtX7v7Vh+SyftZeR5bN81l5Glq92mFUQpFKpoFAoSux3d3eHUqks9zh7e3s4ODjo7VcoFBAEAUql0qAgSCqVwNPTpcrHVZZC4VRj5zYHLJ/ls/YysnyWz9rLyPLVLLMKgsyNVitApco2+nllMikUCieoVDnQaKxvxmGWz/JZexlZPstn7WVk+QynUDhVuobJrIIghUKBjIyMEvuVSiXc3d3LPS4/Px95eXl6tUEqlQoSiaTcYytSk8siaDRaq152geWzfNZeRpbP8ll7GVm+mmVWjY2BgYEl+v5kZGTg/v37Jfr7PHocAFy/fl1vf3x8PBo0aGBwfyAiIiKyXmYVBEVHR+PQoUNQqVTivtjYWEilUkRFRZV5XNu2beHq6opdu3aJ+woKCrBnzx5ER0fXaJ6JiIjIMkkEQRBMnYkiSqUSgwYNQpMmTTB58mQkJSVh8eLFePzxxzF//nwx3YQJE3Dnzh3s3btX3PfFF19g+fLlmDNnDoKDg7Fp0yb8/fff2LlzJxo1amRQfgRBgFZbM2+PTCa1ynbeIiyf5bP2MrJ8ls/ay8jyGUYqlUAikVQqrVn1CXJ3d8eGDRuwcOFCTJ06FS4uLhg5ciRmzZqll06r1UKj0ejte/HFFyEIAr788kukpqYiNDQU69atMzgAAgCJRAKZrHJvpCFMPTSwprF8ls/ay8jyWT5rLyPLV7PMqiaIiIiIqLZYd4hJREREVAYGQURERGSTGAQRERGRTWIQRERERDaJQRARERHZJAZBREREZJMYBBEREZFNYhBERERENolBEBEREdkkBkFERERkkxgEERERkU1iEEREREQ2iUFQLYqLi8Nzzz2HiIgIREVFYcmSJcjPzzd1tozihx9+QEhISIn/li5dauqsGSQhIQHz58/H0KFD0aJFCwwePLjUdNu2bUO/fv0QHh6OIUOG4M8//6zlnBquMmUcN25cqdc1Li7OBDmuvF27duGll15CdHQ0IiIiMHToUGzfvh2PrhdtydevMmW01OsHAPv27cMzzzyDzp07IywsDL169cJ7772HjIwMvXR//PEHhgwZgvDwcPTr1w/ff/+9iXJcdZUp47x580q9hvv37zdhzg2TlZWF6OhohISE4OzZs3qvmepelNfKXyEolUpMmDABAQEBWL58OZKSkrB48WLk5uZi/vz5ps6e0axduxZubm7itq+vrwlzY7irV69i3759aN26NbRabYkvTwD49ddf8X//93+YMmUKOnfujJiYGEybNg3ffvstIiIiaj/TVVSZMgJA27ZtMXfuXL19fn5+tZFFg61fvx4NGzbEvHnz4OnpiUOHDuH//u//cO/ePUybNg2A5V+/ypQRsMzrBwDp6elo1aoVxo0bBw8PD1y9ehXLly/H1atX8eWXXwIA/v33X0ybNg0jR47E66+/jn/++QdvvPEGXFxc0L9/fxOXoGKVKSMANGrUqMQPyqCgoNrObrWtXLkSGo2mxH6T3osC1YpVq1YJERERQlpamrhv8+bNQmhoqHDv3j3TZcxIvv/+eyE4OFhISUkxdVaMQqPRiP+eO3euMGjQoBJp+vbtK8yePVtv35gxY4QXXnihxvNnDJUp4zPPPCNMmjSpNrNlFKV9Dt98802hbdu2Yrkt/fpVpoyWev3KsmXLFiE4OFh8Zj7//PPCmDFj9NLMnj1bGDBggCmyZxSPlrGse9PSXLt2TYiIiBA2bdokBAcHC2fOnBFfM+W9yOawWrJ//35ERkbCw8ND3DdgwABotVocPHjQdBmjUkml5d8at27dwo0bNzBgwAC9/QMHDsThw4ctopmzojJaMi8vrxL7QkNDkZmZiezsbKu4fhWV0RoVPT8LCgqQn5+PI0eOlKjxGThwIOLi4pCYmGiCHFZf8TJak3fffRdjx45FkyZN9Pab+l603qegmYmPj0dgYKDePoVCAW9vb8THx5soV8Y3ePBghIaGolevXli9enWpVZ/WoOiaPXpDBwUFoaCgALdu3TJFtmrE0aNHERERgfDwcDzzzDM4duyYqbNkkOPHj8PX1xeurq5We/2Kl7GIpV8/jUaDvLw8nD9/Hp999hl69uwJPz8/3Lx5EwUFBSWeq0XNRJb0XC2rjEUSEhLQrl07hIWFYcSIEfjtt99MmNuqi42NxZUrVzB16tQSr5n6XmSfoFqiUqmgUChK7Hd3d4dSqTRBjozL29sbr7zyClq3bg2JRII//vgDn3zyCZKSkqyqz1ORomv26DUt2raGawoAHTp0wNChQxEQEIDk5GSsW7cOzz33HDZu3Ig2bdqYOnuV9u+//yImJkbsG2ON1+/RMgLWcf169OiBpKQkAEC3bt3w4YcfArCua1hWGQFd7V54eDiaNm2KjIwMbNq0CVOnTsWnn35qEf2ecnJysHjxYsyaNUsvOC9i6uvIIIiMolu3bujWrZu43bVrVzg4OGDDhg2YMmUKfHx8TJg7MtT06dP1trt3747Bgwdj5cqVWLNmjYlyVTX37t3DrFmz0KlTJ4wfP97U2akRZZXRGq7fF198gZycHFy7dg2ff/45pkyZgq+++srU2TKqssook8kwYcIEvbQ9e/bE2LFjsWzZMosIgj7//HPUqVMHTzzxhKmzUio2h9UShUJRYmgnoIty3d3dTZCjmjdgwABoNBpcvHjR1FkxuqJr9ug1ValUeq9bG2dnZzz22GM4f/68qbNSKSqVCi+++CI8PDywfPlysR+UNV2/sspYGku7fgDQvHlztGnTBqNGjcLKlStx5MgR7N2716quYVllLI1UKkXfvn0RFxeH3NzcWs5p1dy+fRtffvklpk+fjoyMDKhUKrG/WnZ2NrKyskx+HVkTVEsCAwNLtFFnZGTg/v37Jdq0yfwVXbNH+3rFx8fDzs4OjRo1MlXWqFBubi4mT56MjIwMbNmyRW/qBmu5fuWV0RqFhITAzs4ON2/eRM+ePWFnZ4f4+Hi9Wuii56ylPleLl9HSJSYmoqCgAJMmTSrx2vjx49G6dWux6c9U9yJrgmpJdHQ0Dh06JEa3gK6zmFQqRVRUlAlzVnNiYmIgk8nQokULU2fF6Bo1aoSAgADExsbq7Y+JiUFkZCTs7e1NlLOalZ2djb/++gvh4eGmzkq51Go1Zs6cifj4eKxdu7bEfFXWcP0qKmNpLOX6leX06dMoKCiAn58f7O3t0alTJ+zevVsvTUxMDIKCgixiLqTSFC9jabRaLWJjY9GsWTM4OjrWcu6qJjQ0FF9//bXef6+99hoAYMGCBXjrrbdMfi+yJqiWjB07Fhs3bsTUqVMxefJkJCUlYcmSJRg7dqzFTihY3MSJE9GpUyeEhIQAAH7//Xds3boV48ePh7e3t4lzV3U5OTnYt28fAF2VbmZmpniTduzYEV5eXnjllVcwZ84c+Pv7o1OnToiJicGZM2fwzTffmDLrlVZRGYu+XPv06YOGDRsiOTkZX331Fe7fv49PP/3UlFmv0IIFC/Dnn39i3rx5yMzMxKlTp8TXWrRoAXt7e4u/fhWV8cyZMxZ7/QBg2rRpCAsLQ0hICBwdHXHp0iWsW7cOISEh6N27NwDgpZdewvjx4/H2229jwIABOHLkCH755Rd8/PHHJs595VRUxtu3b2PevHkYNGgQGjduDKVSiU2bNuHcuXNYvny5qbNfIYVCgU6dOpX6WsuWLdGyZUsAMOm9KBGEMqaJJaOLi4vDwoULcfLkSbi4uGDo0KGYNWuWRfzqrMi7776LAwcO4N69e9BqtQgICMCoUaMwbtw4SCQSU2evyhITE9GrV69SX/v666/FG3vbtm1Ys2YN7ty5gyZNmmD27Nno0aNHbWbVYBWVsV69enjnnXdw+fJlpKenw8nJCW3atMG0adPQqlWrWs5t1fTs2RO3b98u9bXff/9d/JVtydevojJqNBqLvX6ArrNwTEwMbt68CUEQ0LBhQ/Tp0wcTJ07UG2X0+++/45NPPsH169fRoEEDTJo0CSNHjjRhziuvojKmp6fjtddew4ULF5CSkgI7OzuEhYVh0qRJek2AluTIkSMYP348tm/frlcjaap7kUEQERER2ST2CSIiIiKbxCCIiIiIbBKDICIiIrJJDIKIiIjIJjEIIiIiIpvEIIiIiIhsEoMgIiIiskkMgoiszJEjRxASEoIjR46YOitkwebNm4eePXuaOhtENYpBEJEZunTpEkJCQsTFINevX2+TX0j79u2rleUBkpKSsHz5cly8eLHG/xYRmQ8GQURm6PTp0/Dw8ECTJk0AACdPnkTr1q0rdWyHDh1w5swZdOjQoSazWCv27duHFStW1PjfSU5OxooVKxgEEdkYBkFEZujMmTMIDw8X1107depUpYMgqVQKBwcHSKW8vYmIysOnJJGZUCqVSE1NRWpqKs6cOYNmzZohNTUVV69exb1799C4cWOkpqYiKyur3POU1ido3LhxGDx4MK5du4Zx48ahdevW6NatG9asWVPp/KlUKixatAg9e/ZEWFgYoqOj8d///hepqalimpSUFLz++uvo0qULwsPDMWTIEOzYsUPvPImJiQgJCcG6deuwZcsW9O7dG2FhYXjiiSdw5swZMd28efPw7bffAgBCQkLE/4potVqsX78egwYNQnh4OLp06YL58+dDqVSKaZYtW4bmzZvj8OHDenn4v//7P4SFheHSpUs4cuSIuODma6+9Jv6dH374odz3IykpCa+99hq6dOmCsLAwDBo0CNu3bxdfz83NRf/+/dG/f3/k5uaK+9PT09G1a1eMHTsWGo0GgK75c968eejVqxfCw8MRFRWF1157DWlpaXp/c/ny5QgJCcH169cxZ84ctGvXDp07d8Ynn3wCQRBw9+5dvPTSS2jbti2ioqLw5Zdf6h1f9NmIiYnBRx99hKioKERERGDKlCm4e/duueWt7HsOAGfPnsXEiRPRqVMntGrVCj179sRrr71W4fmJapvc1BkgIp3hw4frrQp+5coVvS+xKVOmiOkWL15c5fMrlUq88MIL6NOnDwYMGIDdu3dj6dKlCA4OxmOPPVbusVlZWXj66acRFxeHJ554Ai1atEBaWhr++OMPJCUlwcvLC7m5uRg3bhxu3ryJp59+Gn5+foiNjcW8efOgUqkwYcIEvXP+8ssvyMrKwpgxYyCRSLB27Vq88sor+O2332BnZ4cxY8YgOTkZBw8exJIlS0rkaf78+dixYwdGjBiBcePGITExEd9++y0uXLiATZs2wc7ODi+99BL+/PNPvPHGG/jpp5/g6uqKAwcOYOvWrZgxYwaaN2+OBw8eYPr06Vi2bBnGjBmDdu3aAQDatm1b5vvx4MEDjB49GhKJBE8//TS8vLywf/9+vPHGG8jMzMSzzz4LR0dHvP/++3jyySfx8ccfi0HAO++8g4yMDLz33nuQyWQAgEOHDuHWrVsYMWIEvL29cfXqVWzduhXXrl3D1q1bxRrBIrNmzUJQUBBeffVV7Nu3D59//jk8PDywefNmdO7cGXPmzMHPP/+M999/H+Hh4SWaRj///HNIJBK8+OKLSElJwYYNG/Dss89i586dcHR0LLPclXnPU1JSMHHiRHh6emLSpElQKBRITEzE3r17y/mEEZmIQERm4d9//xUOHjwofPLJJ0KLFi2Effv2CQcPHhReeOEFYcSIEcLBgweFgwcPClevXi33PP/8848QHBws/PPPP+K+Z555RggODhZ27Ngh7svLyxOioqKEV155pcK8ffrpp0JwcLCwZ8+eEq9ptVpBEARh/fr1QnBwsLBz507xtfz8fGHMmDFCRESEkJGRIQiCINy6dUsIDg4WOnbsKKSnp4tpf/vtNyE4OFj4448/xH0LFiwQgoODS/zNY8eOCcHBwcJPP/2kt3///v0l9l++fFlo2bKl8MYbbwhKpVLo1q2bMGLECKGgoEBMc+bMGSE4OFj4/vvvK3wvBEEQXn/9dSEqKkpITU3V2z9r1iyhXbt2Qk5Ojrjvww8/FJo3by4cO3ZM2LVrlxAcHCysX79e77ji6Yv88ssvQnBwsHDs2DFx37Jly4Tg4GDh//7v/8R9arVaiI6OFkJCQoTVq1eL+5VKpdCqVSth7ty54r6iz0a3bt3E6yEIghATEyMEBwcLGzZsEPfNnTtX6NGjh7hd2fd87969QnBwsHDmzJky3j0i88HmMCIz0a5dO3Tp0gXZ2dkIDw9HdHQ0unTpgrt376JHjx7o0qULunTpgqZNmxp0fmdnZwwdOlTctre3R3h4OG7dulXhsXv27EHz5s3Rp0+fEq8V1VLs378f3t7eGDx4sPianZ0dxo0bh+zsbBw7dkzvuIEDB8Ld3V3cbt++PQBUKj+xsbFwc3NDVFSU2ISYmpqKli1bwtnZWa8pMDg4GNOnT8e2bdswceJEpKWl4f3334dcblhFuCAI2LNnD3r27AlBEPT+fteuXZGRkYHz58+L6adNm4amTZti7ty5WLBgATp27Ijx48frnbN47UteXh5SU1PFPmDFz1WkqPkOAGQyGcLCwiAIgt5+hUKBJk2alPp+Dhs2DK6uruJ2//794e3tjX379pVZ7sq+525ubgCAv/76CwUFBWWej8gcsDmMyAxkZGSIXxiHDx9G586dkZqaCqVSiatXr2LmzJlITU2FnZ2d+CVTVfXq1SvRrOLu7o7Lly+L2/fv39d73c3NDY6Ojrh58yb69u1b7vlv376Nxo0bl+iQHRQUBAC4c+eO3v769euXyAug63tUkYSEBGRkZCAyMrLU11NSUvS2J06ciF9//RVnzpzB7NmzDQ4kASA1NRUqlQpbtmzBli1bykxTxN7eHosWLcLIkSPh4OCARYsWlbgO6enpWLFiBWJiYkrkPSMjo8T5GzRooLft5uYGBwcHeHl5ldifnp5e4vjGjRvrbUskEjRu3FivOfZRlX3PO3bsiH79+mHFihVYv349OnbsiN69e+Pxxx+Hvb19mecnMgUGQURm4OWXX8bRo0fF7cuXL2PDhg3i9tSpUwHovmA2btxo0N8o6n9Snq5du+ptv/feexgxYoRBf8/Q/AiCUOGxWq0WderUwdKlS0t9/dFg4NatW0hISACg62tVHVqtFgAwZMgQDB8+vNQ0xTtwA8Dff/8NQFfLk5CQgEaNGum9PnPmTJw8eRITJ05EaGgonJ2dodVq8cILL5T6fpQ28q8672dlVPY9l0gkWLZsGU6dOoU///wTBw4cwOuvv46vvvoKW7ZsgYuLi1HyQ2QMDIKIzMDcuXOhUqlw8uRJfPbZZ1i1ahXkcjk2btyIpKQkzJkzB4CuiaMmffXVV3rbRTUm/v7+uHr1arnHNmzYEJcvX4ZWq9X7ki6a8PHR2ovKeLTGpIi/vz8OHz6Mtm3bltuRF9B9ec+bNw+urq6YMGECVq1ahX79+unVbJX1d0rj5eUFFxcXaLVadOnSpcL0ly5dwmeffYYRI0bg0qVLePPNN/Hzzz+LNXpKpRKHDx/GK6+8gmnTponH3bhxo9J5qqqigLCIIAhISEgoEbwVV5X3HAAiIiIQERGBWbNm4eeff8acOXMQExODUaNGVTv/RMbCPkFEZiAsLAxdunSBRqNBs2bNxP5AKSkpYl+goqHYNan43+rSpQt8fHwAAH379sWlS5dKHeFTVNMQHR2N+/fvIyYmRnxNrVZj48aNcHZ2NmjyRicnJwAlm8gGDBgAjUaDlStXljhGrVbrpf/qq69w8uRJvPPOO5gxYwbatGmDt99+W6/Jqqy/UxqZTIZ+/fph9+7dpdYqFT9vQUEBXnvtNfj4+OCNN97Ae++9hwcPHmDRokV65ytN8ZpAY/vxxx+RmZkpbsfGxuL+/fuIjo4u85jKvudKpbJE7VNoaCgAID8/3xjZJzIa1gQRmZETJ06gTZs2AHRNJxcuXMDkyZNNnCtdn5rdu3djxowZeOKJJ9CyZUsolUr88ccfWLBgAZo3b44xY8Zgy5YtmDdvHs6fP4+GDRti9+7dOHHiBF5//XW9jriV1bJlSwDAu+++i65du0Imk2HQoEHo2LEjxowZg9WrV+PixYuIioqCnZ0dbty4gdjYWLzxxhvo378/4uLi8Omnn2LEiBHisiOLFy/GsGHDsGDBAnz66acAdLUcCoUCmzdvhouLC5ydndGqVasSzVZFXn31VRw5cgSjR4/GqFGj0LRpUyiVSpw/fx6HDx8WmzY///xzXLx4EevXr4erqyuaN2+OqVOn4pNPPkH//v3x2GOPwdXVFR06dMDatWtRUFAAX19fHDx4EImJiYZcqkpxd3fHU089hREjRohD5Bs3bozRo0eXeUxl3/MdO3Zg06ZN6N27N/z9/ZGVlYWtW7fC1dW13CCLyBQYBBGZCY1Gg9OnT4v9TM6dO4eCggIxKDIlFxcXfPvtt1i+fDn27t2LHTt2oE6dOoiMjISvry8A3QinjRs3YunSpdixYwcyMzPRpEmTavUr6tu3L8aNG4dff/0VP/30EwRBwKBBgwDo5tsJCwvD5s2b8fHHH0Mmk6Fhw4YYMmQI2rZtC41Gg7lz58LT0xOvv/66eM6AgADMnj0b//vf/xATE4OBAwfCzs4OixcvxkcffYS3334barUa7733XplBUN26dbFt2zZ89tln2Lt3LzZt2gQPDw80bdpUbLo8f/48Vq9ejWeeeQadO3cWj500aRJ+//13vPnmm/j111+hUCjw4YcfYuHChfjuu+8gCAKioqKwZs0adOvWzaD3rSJTpkzB5cuX8cUXXyArKwuRkZF46623xBqxslT0ngO6YOns2bOIiYnBgwcP4ObmhlatWmHp0qVlvp9EpiIRjNVrjoiIzNqRI0cwfvx4fPrpp+jfv7+ps0NkcuwTRERERDaJQRARERHZJAZBREREZJPYJ4iIiIhsEmuCiIiIyCYxCCIiIiKbxCCIiIiIbBKDICIiIrJJDIKIiIjIJjEIIiIiIpvEIIiIiIhsEoMgIiIiskkMgoiIiMgm/T/Fsw6c8wx1gwAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "metric = task.get_metric()\n",
        "loss = metric(pred, ys).numpy()\n",
        "\n",
        "sparsity = conf.training.task_kwargs.sparsity if \"sparsity\" in conf.training.task_kwargs else None\n",
        "baseline = {\n",
        "    \"linear_regression\": n_dims,\n",
        "    \"sparse_linear_regression\": sparsity,\n",
        "    \"relu_2nn_regression\": n_dims,\n",
        "    \"decision_tree\": 1,\n",
        "    \"knn_classification\": args.training.task_kwargs[\"k\"] if \"k\" in args.training.task_kwargs else 0\n",
        "}[conf.training.task]\n",
        "\n",
        "plt.plot(loss.mean(axis=0), lw=2, label=\"Transformer\")\n",
        "plt.axhline(baseline, ls=\"--\", color=\"gray\", label=\"zero estimator\")\n",
        "plt.xlabel(\"# in-context examples\")\n",
        "plt.ylabel(\"squared error\")\n",
        "plt.legend()\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "FJ7gN6v4Ua7l"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "V100",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "26eaac5185324eff8fb113c2e5950992": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c61003c168c4b17a26c2e4158fcc2c5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d509a1b559441b38d0128658c9aa3d7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "714f5667409f4901bde3532d57d75b81": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_26eaac5185324eff8fb113c2e5950992",
            "placeholder": "​",
            "style": "IPY_MODEL_80e76cc886a94c6e9b9c5fc223ed9bca",
            "value": " 1/1 [00:00&lt;00:00, 74.58it/s]"
          }
        },
        "7653471e6e0a45ba9bc01ee2e79c3e8d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80e76cc886a94c6e9b9c5fc223ed9bca": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "83aceeb438b04ecc82efb30237f0f586": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ac74f331214c46ed8d05d649f3e6e49d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bc60d092b6314b93bf564a7fb327d246": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7653471e6e0a45ba9bc01ee2e79c3e8d",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_83aceeb438b04ecc82efb30237f0f586",
            "value": 1
          }
        },
        "cd30fdbadb2a4944b24e85c9bb1fd98e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d619083e4d4549c48d5a20c25f72fe30",
              "IPY_MODEL_bc60d092b6314b93bf564a7fb327d246",
              "IPY_MODEL_714f5667409f4901bde3532d57d75b81"
            ],
            "layout": "IPY_MODEL_6c61003c168c4b17a26c2e4158fcc2c5"
          }
        },
        "d619083e4d4549c48d5a20c25f72fe30": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d509a1b559441b38d0128658c9aa3d7",
            "placeholder": "​",
            "style": "IPY_MODEL_ac74f331214c46ed8d05d649f3e6e49d",
            "value": "100%"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}