Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fit_intervention import Latent_Interventions\n",
    "import fit_intervention\n",
    "import pathlib\n",
    "import tensorflow as tf\n",
    "import functions\n",
    "import tensorflow.keras.layers as nn\n",
    "from umap.parametric_umap import ParametricUMAP, load_ParametricUMAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"D:/ICG/Experiments/%s/test\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train, X_test, y_test = functions.get_data(\"cifar10\", path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# os.makedirs(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "optim = tf.keras.optimizers.Adam()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier_model = functions.get_model(\"cifar10\", optim=optim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples\n",
      "Epoch 1/5\n",
      "50000/50000 [==============================] - 8s 166us/sample - loss: 1.8352 - accuracy: 0.4174\n",
      "Epoch 2/5\n",
      "50000/50000 [==============================] - 6s 120us/sample - loss: 1.2199 - accuracy: 0.5691\n",
      "Epoch 3/5\n",
      "50000/50000 [==============================] - 6s 120us/sample - loss: 1.0627 - accuracy: 0.6275\n",
      "Epoch 4/5\n",
      "50000/50000 [==============================] - 6s 122us/sample - loss: 0.9610 - accuracy: 0.6658\n",
      "Epoch 5/5\n",
      "50000/50000 [==============================] - 6s 124us/sample - loss: 0.8818 - accuracy: 0.6927\n"
     ]
    }
   ],
   "source": [
    "history = classifier_model.fit(\n",
    "            X_train,\n",
    "            y_train,\n",
    "            batch_size=64,\n",
    "            epochs=5,\n",
    "            shuffle=True,\n",
    "            verbose=1\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss': [1.8351757162475586,\n",
       "  1.2199492572021484,\n",
       "  1.0626703799819945,\n",
       "  0.9609999384307861,\n",
       "  0.8817546060943604],\n",
       " 'accuracy': [0.4174, 0.5691, 0.6275, 0.66578, 0.69266]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier_model.save(\"%s/classifiermodel-weights-before.h5\"%path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier_model = functions.get_model(\"cifar10\", optim=tf.keras.optimizers.Adam())\n",
    "classifier_model.load_weights(\"%s/classifiermodel-weights-before.h5\"%path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_optim = tf.keras.optimizers.Adam()\n",
    "\n",
    "# get outputs from a certain layer\n",
    "sub_model = tf.keras.Model(inputs=classifier_model.inputs, outputs=classifier_model.get_layer(\"classifier\").output)\n",
    "\n",
    "#print(self.X_train)\n",
    "logits_train = sub_model.predict(X_train)\n",
    "\n",
    "logits_train = tf.reshape(logits_train, [len(logits_train), -1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ParametricUMAP(batch_size=500, loss_report_frequency=1, n_training_epochs=2,\n",
      "               optimizer=<tensorflow.python.keras.optimizer_v2.adam.Adam object at 0x000001C599A1D7C8>)\n",
      "Construct fuzzy simplicial set\n",
      "Thu Oct 29 10:41:32 2020 Finding Nearest Neighbors\n",
      "Thu Oct 29 10:41:32 2020 Building RP forest with 8 trees\n",
      "Thu Oct 29 10:41:34 2020 NN descent for 12 iterations\n",
      "\t 1  /  12\n",
      "\t 2  /  12\n",
      "\t 3  /  12\n",
      "\tStopping threshold met -- exiting after 3 iterations\n",
      "Thu Oct 29 10:41:49 2020 Finished Nearest Neighbor Search\n",
      "Thu Oct 29 10:41:51 2020 Construct embedding\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Embeddings/pumap\\umap\\parametric_umap.py:271: UserWarning: Data should be scaled to the range 0-1 for cross-entropy reconstruction loss.\n",
      "  \"Data should be scaled to the range 0-1 for cross-entropy reconstruction loss.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <function umap_loss.<locals>.loss at 0x000001C58037F798> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: Cell is empty\n",
      "WARNING: AutoGraph could not transform <function umap_loss.<locals>.loss at 0x000001C58037F798> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: Cell is empty\n",
      "Train for 1096 steps\n",
      "Epoch 1/2\n",
      "1096/1096 [==============================] - 8s 8ms/step - loss: 0.2146\n",
      "Epoch 2/2\n",
      "1096/1096 [==============================] - 8s 7ms/step - loss: 0.2048\n",
      "4167/4167 [==============================] - 0s 37us/sample\n",
      "Thu Oct 29 10:42:08 2020 Finished embedding\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ParametricUMAP(batch_size=500, dims=[10],\n",
       "               encoder=<tensorflow.python.keras.engine.sequential.Sequential object at 0x000001C5806DE148>,\n",
       "               loss_report_frequency=1, n_training_epochs=2,\n",
       "               optimizer=<tensorflow.python.keras.optimizer_v2.adam.Adam object at 0x000001C599A1D7C8>)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedder_model = functions.get_model(\"umap\", verbose=1, optim=embedding_optim, batch_size=500, epochs=2, config={})\n",
    "embedder_model.fit(logits_train[::12].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D:/ICG/Experiments/%s/test/embeddermodel\\encoder_model.h5\n",
      "Keras encoder model saved to D:/ICG/Experiments/%s/test/embeddermodel\\encoder_model.h5\n",
      "Keras full model saved to D:/ICG/Experiments/%s/test/embeddermodel\\parametric_model.h5\n",
      "Thu Oct 29 10:42:13 2020 Worst tree score: 0.85601152\n",
      "Thu Oct 29 10:42:13 2020 Mean tree score: 0.86684065\n",
      "Thu Oct 29 10:42:13 2020 Best tree score: 0.87712983\n",
      "Thu Oct 29 10:42:17 2020 Forward diversification reduced edges from 62505 to 24112\n",
      "Thu Oct 29 10:42:21 2020 Reverse diversification reduced edges from 24112 to 24112\n",
      "Thu Oct 29 10:42:23 2020 Degree pruning reduced edges from 25040 to 25040\n",
      "Thu Oct 29 10:42:23 2020 Resorting data and graph based on tree order\n",
      "Thu Oct 29 10:42:23 2020 Compressing index by removing unneeded attributes\n",
      "Thu Oct 29 10:42:23 2020 Building and compiling search function\n",
      "Thu Oct 29 10:42:26 2020 Building and compiling search function\n",
      "Pickle of ParametricUMAP model saved to D:/ICG/Experiments/%s/test/embeddermodel\\model.pkl\n"
     ]
    }
   ],
   "source": [
    "embedder_model.save(\"%s/embeddermodel\"%path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_encoder = embedder_model.encoder\n",
    "embedding_encoder = tf.keras.Model(embedding_encoder.inputs, embedding_encoder.outputs, name=\"embedder\") # rename the model; name is later needed\n",
    "embedding_encoder.trainable = False\n",
    "\n",
    "flat = nn.Flatten()(classifier_model.get_layer(\"classifier\").output) # if conv layer is used as latent layer, we need to flatten the latent space...\n",
    "embedder_out = embedding_encoder(flat)\n",
    "#embedder_out = embedding_encoder(self.classifier_model.get_layer(self.layer_key).output)\n",
    "classifier_embedder_model = tf.keras.Model(classifier_model.input, [classifier_model.output, embedder_out])\n",
    "\n",
    "classifier_embedder_model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(),\n",
    "    loss={\"classifier\": tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \"embedder\": tf.keras.losses.MeanSquaredError()},\n",
    "    loss_weights={\"classifier\": 1, \"embedder\": 1},\n",
    "    metrics={\"classifier\":['accuracy']}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "projected_train = embedder_model.encoder(logits_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "contraction_factors = {\"all\": (1.0/1.2, 1.0/4.0)}\n",
    "\n",
    "labels_train = y_train\n",
    "shifted_train = projected_train.numpy()\n",
    "distinct_labels = list(set(labels_train))\n",
    "\n",
    "for k in contraction_factors:\n",
    "    if k == \"all\":\n",
    "        contraction_factor = contraction_factors[k]\n",
    "        for i in range(len(distinct_labels)):\n",
    "            l = distinct_labels[i]\n",
    "            shifted_train[labels_train == l] = (contraction_factor[0] * shifted_train[(labels_train == l)]) + contraction_factor[1] * shifted_train[(labels_train == l)].mean(axis=0)\n",
    "    else:\n",
    "        shifted_train[labels_train == k] = (contraction_factor[0] * shifted_train[(labels_train == k)]) + contraction_factor[1] * shifted_train[(labels_train == k)].mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples\n",
      "Epoch 1/4\n",
      "50000/50000 [==============================] - 8s 153us/sample - loss: 1.2426 - classifier_loss: 0.7907 - embedder_loss: 0.4518 - classifier_accuracy: 0.7253\n",
      "Epoch 2/4\n",
      "50000/50000 [==============================] - 7s 137us/sample - loss: 1.1770 - classifier_loss: 0.7623 - embedder_loss: 0.4152 - classifier_accuracy: 0.7363\n",
      "Epoch 3/4\n",
      "50000/50000 [==============================] - 7s 139us/sample - loss: 1.1085 - classifier_loss: 0.7249 - embedder_loss: 0.3834 - classifier_accuracy: 0.7456\n",
      "Epoch 4/4\n",
      "50000/50000 [==============================] - 7s 139us/sample - loss: 1.0883 - classifier_loss: 0.6997 - embedder_loss: 0.3887 - classifier_accuracy: 0.7544\n"
     ]
    }
   ],
   "source": [
    "history = classifier_embedder_model.fit(\n",
    "    x=X_train,\n",
    "    y={\"classifier\": y_train, \"embedder\": shifted_train},\n",
    "    batch_size=64,\n",
    "    epochs=4,\n",
    "    shuffle=True,\n",
    "    verbose=1\n",
    ")\n",
    "\n",
    "classifier_embedder_model.save(\"%s/classifiermodel-weights-after-embedding.h5\"%path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss': [1.242590708732605,\n",
       "  1.1770126445007325,\n",
       "  1.1085271305084228,\n",
       "  1.0882534571838378],\n",
       " 'classifier_loss': [0.7907495, 0.76225173, 0.7248937, 0.69971544],\n",
       " 'embedder_loss': [0.4518156, 0.41520947, 0.38342008, 0.38868737],\n",
       " 'classifier_accuracy': [0.72526, 0.73628, 0.74558, 0.75438]}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = [1.8351757162475586,\n",
    "  1.2199492572021484,\n",
    "  1.0626703799819945,\n",
    "  0.9609999384307861,\n",
    "  0.8817546060943604,\n",
    "  0.7907495, 0.76225173, 0.7248937, 0.69971544\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1c5b3c6ad48>]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgGUlEQVR4nO3deXSV9b3v8fd3ZyAJmYDMCQmDDCJDgqngBI4VEAW6tC1Ot9ZTSnvs0dN1Vz2ec+89t6d3rXt67bn3WNtqrVPVisejTA6F2kFBBZUhyCRTkCEJSSCQARIy/e4fe0MBAwlkh2cPn9daWSR7P9nPB4wfHr7Ps3+POecQEZHw5/M6gIiIBIcKXUQkQqjQRUQihApdRCRCqNBFRCJErFc7zsjIcEOGDPFq9yIiYWnt2rUHnXOZXT3nWaEPGTKENWvWeLV7EZGwZGZ7zvacRi4iIhFChS4iEiFU6CIiEUKFLiISIVToIiIRQoUuIhIhVOgiIhEi7Ap9Z00j//LmFlrbO72OIiISUsKu0PfVNfPch7t5f3ut11FEREJK2BX6NSMyGNg/nsVlFV5HEREJKWFX6HExPmaOz+WPW6ppbGnzOo6ISMjottDN7DkzqzGzTWd5Ps3M3jSzDWa22czuD37M080qzuN4eyfLN1f39a5ERMJGT47QXwCmneP5vwW2OOcmANcB/2Zm8b2PdnYTCwcweGAiSzR2ERE5qdtCd86tAOrOtQmQYmYGJAe2bQ9OvK6ZGbMm5PPhzoPUNLb05a5ERMJGMGbovwAuBSqBjcBDzrkuryk0s3lmtsbM1tTW9u4qldkleXQ6eHNDVa9eR0QkUgSj0G8ByoA8oBj4hZmldrWhc+5p51ypc640M7PL9dl77JKsFC7LS9XYRUQkIBiFfj+w0PntBHYDo4Pwut2aXZzPZ/vrKa9tuhi7ExEJacEo9L3AjQBmlg2MAsqD8Lrdum1CHmawuKzyYuxORCSk9eSyxQXAKmCUme03swfMbL6ZzQ9s8hPgKjPbCPwJeMQ5d7DvIv9VTloCVw4bxJKyCpxzF2OXIiIhq9t7ijrn5nbzfCXw1aAlOk+zi/P50RufUbbvCCWFA7yKISLiubB7p+iZpo3LIT7WxxKNXUQkyoV9oacmxHHj6Cze+qyS9g6twCgi0SvsCx1gVnE+B5ta+WDnRRndi4iEpIgo9OtHZ5KaEKuxi4hEtYgo9H6xMcwYl8vyzQdobu3wOo6IiCciotDBP3Y51trBu1u1AqOIRKeIKfRJQweSk5rAkvVaCkBEolPEFLrPZ9xenMf722upO9rqdRwRkYsuYgod/De+aO90vL1RKzCKSPSJqEIfk5vKiKxkjV1EJCpFVKGbGbNL8lmz5zD76o55HUdE5KKKqEIHuH1CHgBLN+iadBGJLhFX6IMHJlFaNIDF67UCo4hEl4grdIBZJfnsqGliS1WD11FERC6aiCz0W8flEuszLQUgIlElIgt9YP94po7MZGlZJR2dGruISHSIyEIH/9jlQEMLH+8+5HUUEZGLImIL/eZLs+kfH8OS9Rq7iEh0iNhCT4yP4ZbLcnhnUxUtbVqBUUQiX8QWOvjHLo0t7by3rcbrKCIifS6iC/3q4YPISI7X1S4iEhUiutBjY3zMHJ/Hnz6voaGlzes4IiJ9KqILHfwrMLa2d7Js4wGvo4iI9KmIL/TiwekUDUpicZlWYBSRyBbxhW5mzCrOZ1X5IQ7Ut3gdR0Skz0R8oQPMLs7DOXhTKzCKSATrttDN7DkzqzGzTefY5jozKzOzzWb2fnAj9t6wzGTGF6Rp7CIiEa0nR+gvANPO9qSZpQO/Am53zl0G3BmUZEE2qzifzZUN7Kxp9DqKiEif6LbQnXMrgLpzbHIXsNA5tzewfUi+i+e2Cbn4DBZrKQARiVDBmKGPBAaY2XtmttbM7jvbhmY2z8zWmNma2traIOy657JSErj6kgyWbNCNL0QkMgWj0GOBy4FbgVuA/25mI7va0Dn3tHOu1DlXmpmZGYRdn59Zxfnsq2tm3d7DF33fIiJ9LRiFvh9Y5pw76pw7CKwAJgThdYPulsuy6Rfr09hFRCJSMAp9CXCtmcWaWRIwCdgahNcNupSEOG4ak83bG6to6+j0Oo6ISFD15LLFBcAqYJSZ7TezB8xsvpnNB3DObQWWAZ8BnwDPOOfOeomj12YX51N3tJWVOy7uDF9EpK/FdreBc25uD7Z5DHgsKIn62NSRmaQnxbF4fSU3jM72Oo6ISNBExTtFTxUf62PGuFze3VLN0ePtXscREQmaqCt08I9dmts6eHdLtddRRESCJioLvbRoAPnpiVoKQEQiSlQWus9n3DYhj5U7DnKw6bjXcUREgiIqCx1gdkkeHZ2Otz+r8jqKiEhQRG2hj85JZXROisYuIhIxorbQwb8UwPq9R9hz6KjXUUREei2qC/324jwAlpRpKQARCX9RXej56YlcMXQgi8u0AqOIhL+oLnTwX5NeXnuUTRUNXkcREemVqC/0GeNyiIsxnRwVkbAX9YWenhTPdaOyeHNDJR2dGruISPiK+kIH/9ilpvE4q3Yd8jqKiMgFU6EDN16aRXK/WI1dRCSsqdCBhLgYpo3NYdmmA7S0dXgdR0TkgqjQA2YX59N0vJ0/ba3xOoqIyAVRoQdcOXwQWSn9NHYRkbClQg+ICazA+N62GuqPtXkdR0TkvKnQTzGrOI+2Dsc7m7QCo4iEHxX6KcblpzEsoz+L12vsIiLhR4V+CjNjVnE+H++uo/JIs9dxRETOiwr9DLMCKzAu3aAVGEUkvKjQzzAkoz/Fg9M1dhGRsKNC78Ls4jw+P9DItgONXkcREekxFXoXZk7II8anFRhFJLyo0LuQkdyPay7JYGlZJZ1agVFEwkS3hW5mz5lZjZlt6ma7r5hZh5ndEbx43pldkkfFkWbW7DnsdRQRkR7pyRH6C8C0c21gZjHAT4HlQcgUEr46JofEuBiNXUQkbHRb6M65FUBdN5v9AHgDiJiVrfr3i+XmMdm8s7GK1vZOr+OIiHSr1zN0M8sH5gBP9WDbeWa2xszW1NbW9nbXfW52SR5HjrXx/vbQzyoiEoyTov8OPOKc63Yhcefc0865UudcaWZmZhB23beuHZHJwP7xGruISFiIDcJrlAKvmhlABjDDzNqdc4uD8Nqeiovxceu4XF5bs4/GljZSEuK8jiQicla9PkJ3zg11zg1xzg0BXge+HwllfsLskjyOt3eyfHO111FERM6pJ5ctLgBWAaPMbL+ZPWBm881sft/H897EwgEUDEhkicYuIhLiuh25OOfm9vTFnHPf6lWaEORfgTGPJ9/bRU1jC1kpCV5HEhHpkt4p2gOzi/PpdPDWBt34QkRClwq9B0ZkpzAmN1VjFxEJaSr0HppdkseG/fXsPnjU6ygiIl1SoffQ7RPyMUPrpItIyFKh91BOWgKThw5iSVkFzmkFRhEJPSr08zC7JI8vDh1jw/56r6OIiHyJCv08TBubS3yMT2MXEQlJKvTzkJYYxw2js3jrs0raO7QCo4iEFhX6eZpdksfBplY+3HXI6ygiIqdRoZ+n60ZlkZIQyxKNXUQkxKjQz1NCXAwzxuayfPMBmlu7XTFYROSiUaFfgFkleRxt7eDdrVqBUURChwr9AkweOoic1ASNXUQkpKjQL4DPZ9xenMf722upO9rqdRwREUCFfsFmFefR3ul4Z6NWYBSR0KBCv0BjclO5JCtZKzCKSMhQoV8gM2N2cR6ffnGY/YePeR1HRESF3huzivMBWFJW6XESEREVeq8MHpjE5UUDtAKjiIQEFXovzS7OY3t1E1urGr2OIiJRToXeS7eOzyPWZzo5KiKeU6H30sD+8UwZmcnSDZV0dmrsIiLeUaEHwaziPKrqW/h4d53XUUQkiqnQg+DmMdkkxcdo7CIinlKhB0FSfCy3XJbDOxuraGnTCowi4g0VepB8vXQwDS3tfO1XH7GzpsnrOCIShbotdDN7zsxqzGzTWZ6/28w+C3x8ZGYTgh8z9F05fBDP3FdKVX0zM59YyYJP9uradBG5qHpyhP4CMO0cz+8GpjrnxgM/AZ4OQq6wdNOYbJY9PIXLiwbw6MKNfP936zhyTKsxisjF0W2hO+dWAGe9fMM595Fz7nDgy9VAQZCyhaXs1ARe+vYkHp0+mne3VDP98ZWsLtf9R0Wk7wV7hv4A8PuzPWlm88xsjZmtqa2tDfKuQ4fPZ3x36nAWfv8qEuJimPub1fxs+TbaOjq9jiYiESxohW5m1+Mv9EfOto1z7mnnXKlzrjQzMzNYuw5Z4wvSeesH13DHxAJ+8ZedfP3Xq9h7SCszikjfCEqhm9l44BlglnNO84VT9O8Xy2N3TuCJuSXsrGlixs9Xsli3rhORPtDrQjezQmAhcK9zbnvvI0Wm2ybk8fuHrmV0TgoP/0cZf/8fZTS2tHkdS0QiSE8uW1wArAJGmdl+M3vAzOab2fzAJv8DGAT8yszKzGxNH+YNawUDknh13mQevmkES8oqmPHzlazbe7j7bxQR6QHz6lrp0tJSt2ZN9Hb/mi/qeOjVMg40tPDDm0cyf+pwYnzmdSwRCXFmttY5V9rVc3qnqEdKhwzknYeuZfrYHB5bvo27frOayiPNXscSkTCmQvdQWmIcT8wt4bE7xrOxop7pj69k2aYqr2OJSJhSoXvMzLizdDBv/921FA1KYv7L63h04UaOtbZ7HU1EwowKPUQMzejP6/Ov4rtTh7Hgk73c9sQHbK6s9zqWiIQRFXoIiY/18ej0S3n5gUk0trQz55cf8czKct0JSUR6RIUegq4ZkcGyh6cwZWQm/+vtrdz/wqfUNh73OpaIhDgVeoga2D+e39x3OT+ZdRmryw8x/fEV/GVbjdexRCSEqdBDmJlx75VDWPrgNQzq34/7n/+UH7+5mePtuiuSiHyZCj0MjMpJYcmDV/Otq4bw/IdfMPuXH7GzptHrWCISYlToYSIhLob/eftlPPtfSqluaGHmEx/wu4/36K5IInKSCj3M3HhpNsseupavDBnIPy3axPyX13L4qO6KJCIq9LCUlZrAb++/gn+acSl//ryG6Y+v5KNdB72OJSIeU6GHKZ/P+M6UYSz83tUkxsdw9zMf83+Wfa67IolEMRV6mBtXkMZbP7iGr18+mF+9t4s7nlrFnkNHvY4lIh5QoUeA/v1i+ekd4/nlXRPZXdvEjMdXsnDdfq9jichFpkKPILeOz+X3D09hTF4qP3xtAw+/up4G3RVJJGqo0CNMfnoiC74zmR/ePJI3P6vi1p+vZO0e3RVJJBqo0CNQbIyPv7txBK99dzLOwZ1PfcTDr65na1WD19FEpA+p0CPY5UX+uyJ9++qh/GFLNdMfX8m3nv+E1eWH9IYkkQike4pGiSPHWnlp1R5e+OgLDh1tpXhwOvOnDuerY7Lx6V6mImHjXPcUVaFHmebWDl5fu4+nV5azr66Z4Zn9+e6U4cwqyaNfbIzX8USkGyp0+ZL2jk7e2XSAJ9/bxdaqBrJT+/HANUOZe0UhKQlxXscTkbNQoctZOedYseMgT723i1Xlh0hJiOXeyUXcf/VQMlP6eR1PRM6gQpce2bDvCE+9v4tlmw8QF+PjjssLmHftMIZk9Pc6mogEqNDlvJTXNvGbleW8sbaC9s5Opo/L5XtThzM2P83raCJRr1eFbmbPATOBGufc2C6eN+BxYAZwDPiWc25dd6FU6KGvpqGF5z78gt+t3kPj8XauuSSD+VOHc/Ulg/D/ZxeRi+1chd6T69BfAKad4/npwIjAxzzgyfMNKKEpKzWBf5g+mg8fvYFHpo1mW3Uj9zz7Mbf94gPe+qySjk5dyy4SSrotdOfcCqDuHJvMAl50fquBdDPLDVZA8V5qQhzfu244K390Pf/7a+M4eryDB19Zzw3/9h4vr95DS5vucSoSCoLxTtF8YN8pX+8PPCYRJiEuhrlXFPLHH07lybsnkp4Yx39bvIlrfvpnfvmXndQ3ayEwES/FBuE1uhqmdvlvcTObh38sQ2FhYRB2LV6I8RnTx+UybWwOq8oP8dT75Ty2fBtPvreLuyYV8u2rh5KTluB1TJGoE4xC3w8MPuXrAqCyqw2dc08DT4P/pGgQ9i0eMjOuGp7BVcMz2FRRz69XlPPMynKe/3A3c0rymTdlOJdkJXsdUyRqBGPkshS4z/wmA/XOuaogvK6EkbH5aTwxt4T3/uv1fPMrhSwpq+Tm//c+815cw7q9Wr5X5GLoyWWLC4DrgAygGvhnIA7AOfdU4LLFX+C/EuYYcL9zrtvrEXXZYmQ72HSc3370BS+u2kN9cxtXDB3I96YO57pRmbrkUaQX9MYi8czR4+0s+GQvz36wm6r6FkbnpDB/6nBmjs8lNkarN4ucLxW6eK61vZOlGyr59fu72FHTRH56It+5dihzJhaQlqjFwER6SoUuIaOz0/Gnz2t46v1drN1zmPhYHzddmsWckgKmjswkPlZH7SLncq5CD8ZVLiI95vMZN4/J5uYx2WzYd4RF6ytYuqGSdzYeYEBSHLdNyGNOST7Fg9M1axc5TzpCF8+1dXSyYnstC9dX8O6WalrbOxma0Z/ZxfnMKcmncFCS1xFFQoZGLhI2Glra+P3GKhatr2B1uX/FidKiAcyZmM/McXmkJWneLtFNhS5hqeJIM4vXV7BofQU7a5qIj/Fxw+gs5kzM5/pRWZq3S1RSoUtYc86xqaKBhev38+aGSg42tZKeFMet43L52sR8JhYO0LxdooYKXSJGe0cnK3ceZNG6Cv6w5QAtbZ0UDUo6OW/X3ZUk0qnQJSI1trSxbNMBFpdV8NGuQzgHJYXpfK0kn5nj8xjQP97riCJBp0KXiFdV38ySskoWratgW3UjcTHGdaOy+FpJPjdcmkW/2BivI4oEhQpdooZzji1VDSxaV8GSDZXUNh4nNSGWW8fnMqekgNKiAfh8mrdL+FKhS1Tq6HR8uPMgi9ZXsGzTAZrbOigYkMicEv+8fVimlvaV8KNCl6h39Hg7yzcfYNH6Cj7ceZBOBxMGn5i35zIouZ/XEUV6RIUucorqhhaWllWycH0FW6saiPUZU0dmMmdiPjddmk1CnObtErpU6CJn8fkB/7x9cVkF1Q3HSekXy4xxuXzzisFaT0ZCkgpdpBsdnY7V5YdYuK6C32+q4lhrB+Py07j3yiJun5Cno3YJGSp0kfPQdLydRev28+KqPeyoaSItMY6vlxZw96QivXFJPKdCF7kAzjk+3l3HS6v2sHzzAdo7HVNHZnLv5CKuH51FjC5/FA+o0EV6qbqhhQWf7OWVj/dS03icggGJ3D2piK+XFugKGbmoVOgiQdLW0cm7W6p5cdUXrC6vIz7Gx8zxudx7ZZFOospFoUIX6QM7qht5afUeFq6roOl4O2PzU7lv8hBum5BHYrxOokrfUKGL9KGm4+0sWl/BS6u+YHu1/yTqnZcXcM9knUSV4FOhi1wEzjk+2V3Hi6v3sHyT/yTqlJGZ3KeTqBJEukm0yEVgZkwaNohJwwZR3dDCq5/s45VP9vA3L64hPz2RuycX8o3SwTqJKn1GR+gifaito5M/bqnmxVV7WFV+iPgYH7eOz+WeyUVMLNRJVDl/GrmIhIAd1Y28vHoPbwROol6Wl8p9VxZx+4R8nUSVHut1oZvZNOBxIAZ4xjn3r2c8nwa8DBTiH+P8zDn3/LleU4Uu0erMk6ipCbHcWTqYeyYXMVQnUaUbvSp0M4sBtgM3A/uBT4G5zrktp2zzj0Cac+4RM8sEtgE5zrnWs72uCl2i3YmTqC+t3sOywEnUa0dkcN+VQ7hBJ1HlLHp7UvQKYKdzrjzwYq8Cs4Atp2zjgBTzDwSTgTqgvVepRSLcqSdRaxpaePXTfbzy8V6+EziJetekQr7xlcFk6CSq9FBPjtDvAKY55/4m8PW9wCTn3IOnbJMCLAVGAynAN5xzb3fxWvOAeQCFhYWX79mzJ1i/D5GI0NVJ1Bnjcrj3yiE6iSpA74/Qu/oJOvNvgVuAMuAGYDjwrpmtdM41nPZNzj0NPA3+kUsP9i0SVeJifEwfl8v0cbnsrGnkpVX+k6iLyyq5NDeV0qIBFA5MYvDARAYPTGLwwCRSE+K8ji0hoieFvh8YfMrXBUDlGdvcD/yr8x/u7zSz3fiP1j8JSkqRKHRJVgo/njWWH00bzaL1Fbyxbj+LyypobDl9mpmWGHd6yQ/wF33hwCTy0xOJj/V59DuQi60nhf4pMMLMhgIVwDeBu87YZi9wI7DSzLKBUUB5MIOKRKv+/WK5Z3IR90wuAqD+WBv7Dh9jb90x9tUdC3zezOdVjfxxSw2tHZ0nv9cMclMTKAgU/YniLwwc3Wcm98Onk68Ro9tCd861m9mDwHL8ly0+55zbbGbzA88/BfwEeMHMNuIf0TzinDvYh7lFolZaUhxpSWmMzU/70nOdnY7qxhb21TWfVvj76o7x4c6DvNHQctr28bE+Bg9IPHlE7z+61zgnXOmNRSJRpKWtg4ojzf6irzvGvsP+z0+Uf8MZ45z0pLjTS/7kUb7GOV7RWi4iAkBCXAzDM5MZnpnc5fMnxjknS/7wMfb1YJxzSVYyo7JTGJHt/1Xr1XhDhS4iJ3U3zqlpPH7yaP5E4e89dIy3P6vilea9J7fNSI5nZHbKyY9ROcmMyE7RCKePqdBFpEd8PiMnLYGctASuGDrwtOecc9Q2HmdbdSPbq5vYfqCRbdWN/OeafRxt7Ti5XW5aQqDgT5R9MiOyUrSWTZCo0EWk18yMrNQEslITuHZE5snHOzsdFUea2VHTyLYDTWyvbmTbgUZWlR+itb0z8L1QODDpZMGfKPxhGcma0Z8nFbqI9Bmfz05eMXPD6OyTj7d3dLK37lig4P1Fv726kT9/XkNHp/9CjVifMSSjP6POGNsUDUwiNkZF3xUVuohcdLExPoZlJjMsM5lpY//6+PH2DnYfPMq2A43sqG5iW3UjmyrreWdTFScuyIuP9XFJZjKjcv56EnZkdgr56YlRf029Cl1EQka/2BhG56QyOif1tMePtbazs6bJP58PjG1Wlx9i0fqKk9skxccwIjuFUYGxzcjsFIoGJZGTlkC/2OiY0avQRSTkJcXHMr4gnfEF6ac9Xt/cxs4z5vN/2lrDa2v2n7ZdRnI/8tITyEtLJDc9gfz0RHLTEv2PpSdGzDtmVegiErbSEuO4vGgglxedftXNwabjbK9upOJwM5VHWqiqb6biSDM7a5tYsaOWY6dceQMQF2Nkp/rLPS/N/2tueiL56QmB4k8kNSE25Fe7VKGLSMTJSO531nXknXM0NLdTcaSZqvpmKo80U1nfQuWRZqqOtPDpF4epbqiivfP0d9H3j4/psuhP/AWQk5ZAQpy3ox0VuohEFTMLvIEqjjF5qV1u09Hpv66+MlD4VUdaTv4FUFXfwpbKeg42ffmGbBnJ8SdHOblpif7RTvqJI/9EMlP69emdqFToIiJniDnlTVQTCwd0uU1LWwcH6lsCpd9C1ZHmk5+X1x7lgx0HT3tTFfgvxcxOTeBbVw3hO1OGBT23Cl1E5AIkxMUwJKM/Q85yY2/nHA0t7X8d6xwJjHXqW8hK7Zu1blToIiJ9wMxIS4wjLTHuS5dh9hW93UpEJEKo0EVEIoQKXUQkQqjQRUQihApdRCRCqNBFRCKECl1EJEKo0EVEIoQ557rfqi92bFYL7LnAb88ADgYxTrCEai4I3WzKdX6U6/xEYq4i51xmV094Vui9YWZrnHOlXuc4U6jmgtDNplznR7nOT7Tl0shFRCRCqNBFRCJEuBb6014HOItQzQWhm025zo9ynZ+oyhWWM3QREfmycD1CFxGRM6jQRUQiRNgVuplNM7NtZrbTzP7B6zwAZvacmdWY2Savs5zKzAab2V/MbKuZbTazh7zOBGBmCWb2iZltCOT6sdeZTmVmMWa23sze8jrLCWb2hZltNLMyM1vjdZ4TzCzdzF43s88DP2dXhkCmUYE/pxMfDWb2sNe5AMzs7wM/85vMbIGZJQT19cNphm5mMcB24GZgP/ApMNc5t8XjXFOAJuBF59xYL7OcysxygVzn3DozSwHWArND4M/LgP7OuSYziwM+AB5yzq32MtcJZvZDoBRIdc7N9DoP+AsdKHXOhdSbZMzst8BK59wzZhYPJDnnjngc66RAZ1QAk5xzF/pGxmBlycf/sz7GOddsZq8B7zjnXgjWPsLtCP0KYKdzrtw51wq8CszyOBPOuRVAndc5zuScq3LOrQt83ghsBfK9TQXOrynwZVzgIySOLMysALgVeMbrLKHOzFKBKcCzAM651lAq84AbgV1el/kpYoFEM4sFkoDKYL54uBV6PrDvlK/3EwIFFQ7MbAhQAnzscRTg5FijDKgB3nXOhUQu4N+BHwGdHuc4kwP+YGZrzWye12EChgG1wPOBEdUzZtb1HZO9801ggdchAJxzFcDPgL1AFVDvnPtDMPcRboVuXTwWEkd2oczMkoE3gIedcw1e5wFwznU454qBAuAKM/N8VGVmM4Ea59xar7N04Wrn3ERgOvC3gTGf12KBicCTzrkS4CgQEue1AAIjoNuB//Q6C4CZDcA/URgK5AH9zeyeYO4j3Ap9PzD4lK8LCPI/WSJNYEb9BvA759xCr/OcKfBP9PeAad4mAeBq4PbAvPpV4AYze9nbSH7OucrArzXAIvzjR6/tB/af8q+r1/EXfKiYDqxzzlV7HSTgJmC3c67WOdcGLASuCuYOwq3QPwVGmNnQwN++3wSWepwpZAVOPj4LbHXO/V+v85xgZplmlh74PBH/D/rnnoYCnHOPOucKnHND8P9s/dk5F9QjqAthZv0DJ7UJjDS+Cnh+RZVz7gCwz8xGBR66EfD0hPsZ5hIi45aAvcBkM0sK/L95I/7zWkETG8wX62vOuXYzexBYDsQAzznnNnscCzNbAFwHZJjZfuCfnXPPepsK8B9x3gtsDMyrAf7ROfeOd5EAyAV+G7gCwQe85pwLmUsEQ1A2sMjfAcQCrzjnlnkb6aQfAL8LHGCVA/d7nAcAM0vCfzXcd73OcoJz7mMzex1YB7QD6wnyEgBhddmiiIicXbiNXERE5CxU6CIiEUKFLiISIVToIiIRQoUuIhIhVOgiIhFChS4iEiH+PxJHB4x2HplGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fit_intervention import Latent_Interventions\n",
    "import fit_intervention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# could also load this from a json file\n",
    "meta_data = {}\n",
    "meta_data[\"dataset\"] = \"cifar10\" #cifar10\n",
    "meta_data[\"model_name\"] = \"cifar10\" #cifar10\n",
    "\n",
    "# classifier config\n",
    "meta_data[\"epochs\"] = 5#10\n",
    "meta_data[\"post_epochs\"] = 4\n",
    "meta_data[\"optim\"] = \"Adam\"\n",
    "meta_data[\"optim_config\"] = tf.keras.optimizers.Adam().get_config()\n",
    "meta_data[\"batch_size\"] = 64#*8\n",
    "\n",
    "# intervention config\n",
    "meta_data[\"layer_key\"] = 'classifier' # which latent layer output should be used for embedding\n",
    "meta_data[\"contraction_factors\"] = {\"all\": (1.0/1.2, 1.0/4.0)} # how much should the class points in the embedding be shifted to their center of mass\n",
    "meta_data[\"shift_factors\"] = {} # how much should the class points be shifted in a certain direction\n",
    "\n",
    "# embedder config\n",
    "meta_data[\"embedding_approach\"] = \"umap\" # one of: [\"umap\", \"tsne\"]\n",
    "meta_data[\"embedding_optim\"] = \"Adam\"\n",
    "meta_data[\"embedding_optim_config\"] = tf.keras.optimizers.Adam().get_config()\n",
    "meta_data[\"embedding_epochs\"] = 2#10 #1 for umap #10 for tsne\n",
    "meta_data[\"embedding_batch_size\"] = 500\n",
    "meta_data[\"embedding_subset\"] = 12 # subset that should be selected for training the embedding\n",
    "meta_data[\"embedding_weight\"] = 0.5 # how much influence should the embedding loss have on training the classifier\n",
    "# meta_data[\"embedding_conf\"] = {'layer_dims': [300,100], 'alpha': 1.0, 'preplexities': 100, 'do_pretrain': True, 'beta_batch_size':1000} #500} # embedding approach speciffic configuration\n",
    "meta_data[\"embedding_conf\"] = {} # for unet we dont need any special config for now\n",
    "\n",
    "\n",
    "meta_data[\"experiment_number\"] = 0\n",
    "# path = pathlib.Path().absolute()\n",
    "# path = \"D:/ICG/Experiments/%s/Experiment-umap-embepochs/Experiment-embepoch\"%meta_data[\"dataset\"]\n",
    "meta_data[\"base_path\"] = path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--Warning! Experiment path already exists. By calling further methods, the current files will be overwritten.--\n",
      "saved file to D:/ICG/Experiments/%s/test/meta_data.json\n",
      "--fitting classifier basemodel--\n",
      "Train on 50000 samples\n",
      "Epoch 1/5\n",
      "50000/50000 [==============================] - 6s 130us/sample - loss: 1.7981 - accuracy: 0.3848\n",
      "Epoch 2/5\n",
      "50000/50000 [==============================] - 6s 123us/sample - loss: 1.2703 - accuracy: 0.5488\n",
      "Epoch 3/5\n",
      "50000/50000 [==============================] - 6s 122us/sample - loss: 1.1010 - accuracy: 0.6131\n",
      "Epoch 4/5\n",
      "50000/50000 [==============================] - 6s 122us/sample - loss: 0.9843 - accuracy: 0.6552\n",
      "Epoch 5/5\n",
      "50000/50000 [==============================] - 6s 121us/sample - loss: 0.8999 - accuracy: 0.6874\n",
      "saved file to D:/ICG/Experiments/%s/test/classifiermodel-history-before.json\n",
      "--fitting embedder model--\n",
      "ParametricUMAP(batch_size=500, loss_report_frequency=1, n_training_epochs=2,\n",
      "               optimizer=<tensorflow.python.keras.optimizer_v2.adam.Adam object at 0x000001C5B30221C8>)\n",
      "Construct fuzzy simplicial set\n",
      "Thu Oct 29 10:46:06 2020 Finding Nearest Neighbors\n",
      "Thu Oct 29 10:46:06 2020 Building RP forest with 8 trees\n",
      "Thu Oct 29 10:46:06 2020 NN descent for 12 iterations\n",
      "\t 1  /  12\n",
      "\t 2  /  12\n",
      "\t 3  /  12\n",
      "\tStopping threshold met -- exiting after 3 iterations\n",
      "Thu Oct 29 10:46:07 2020 Finished Nearest Neighbor Search\n",
      "Thu Oct 29 10:46:07 2020 Construct embedding\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Embeddings/pumap\\umap\\parametric_umap.py:271: UserWarning: Data should be scaled to the range 0-1 for cross-entropy reconstruction loss.\n",
      "  \"Data should be scaled to the range 0-1 for cross-entropy reconstruction loss.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <function umap_loss.<locals>.loss at 0x000001C584F28AF8> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: Cell is empty\n",
      "WARNING: AutoGraph could not transform <function umap_loss.<locals>.loss at 0x000001C584F28AF8> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: Cell is empty\n",
      "Train for 1080 steps\n",
      "Epoch 1/2\n",
      "1080/1080 [==============================] - 9s 9ms/step - loss: 0.2229\n",
      "Epoch 2/2\n",
      "1080/1080 [==============================] - 8s 7ms/step - loss: 0.2074\n",
      "4167/4167 [==============================] - 0s 34us/sample\n",
      "Thu Oct 29 10:46:24 2020 Finished embedding\n",
      "D:/ICG/Experiments/%s/test/embeddermodel\\encoder_model.h5\n",
      "Keras encoder model saved to D:/ICG/Experiments/%s/test/embeddermodel\\encoder_model.h5\n",
      "Keras full model saved to D:/ICG/Experiments/%s/test/embeddermodel\\parametric_model.h5\n",
      "Thu Oct 29 10:46:24 2020 Worst tree score: 0.85313175\n",
      "Thu Oct 29 10:46:24 2020 Mean tree score: 0.86369090\n",
      "Thu Oct 29 10:46:24 2020 Best tree score: 0.87784977\n",
      "Thu Oct 29 10:46:24 2020 Forward diversification reduced edges from 62505 to 23879\n",
      "Thu Oct 29 10:46:24 2020 Reverse diversification reduced edges from 23879 to 23879\n",
      "Thu Oct 29 10:46:24 2020 Degree pruning reduced edges from 24928 to 24928\n",
      "Thu Oct 29 10:46:24 2020 Resorting data and graph based on tree order\n",
      "Thu Oct 29 10:46:24 2020 Compressing index by removing unneeded attributes\n",
      "Thu Oct 29 10:46:24 2020 Building and compiling search function\n",
      "Thu Oct 29 10:46:26 2020 Building and compiling search function\n",
      "Pickle of ParametricUMAP model saved to D:/ICG/Experiments/%s/test/embeddermodel\\model.pkl\n",
      "--fitting classifier with interventions--\n",
      "Train on 50000 samples\n",
      "Epoch 1/4\n",
      "49920/50000 [============================>.] - ETA: 0s - loss: 1.3587 - classifier_loss: 0.8303 - embedder_loss: 0.5283 - classifier_accuracy: 0.7122\n",
      "Epoch 00001: saving model to D:/ICG/Experiments/%s/test/classifiermodel-weights-after-embedding-epoch01.hdf5\n",
      "50000/50000 [==============================] - 8s 158us/sample - loss: 1.3584 - classifier_loss: 0.8303 - embedder_loss: 0.5283 - classifier_accuracy: 0.7124\n",
      "Epoch 2/4\n",
      "49920/50000 [============================>.] - ETA: 0s - loss: 1.2542 - classifier_loss: 0.7853 - embedder_loss: 0.4689 - classifier_accuracy: 0.7256\n",
      "Epoch 00002: saving model to D:/ICG/Experiments/%s/test/classifiermodel-weights-after-embedding-epoch02.hdf5\n",
      "50000/50000 [==============================] - 7s 135us/sample - loss: 1.2543 - classifier_loss: 0.7854 - embedder_loss: 0.4687 - classifier_accuracy: 0.7255\n",
      "Epoch 3/4\n",
      "49728/50000 [============================>.] - ETA: 0s - loss: 1.2343 - classifier_loss: 0.7607 - embedder_loss: 0.4737 - classifier_accuracy: 0.7342\n",
      "Epoch 00003: saving model to D:/ICG/Experiments/%s/test/classifiermodel-weights-after-embedding-epoch03.hdf5\n",
      "50000/50000 [==============================] - 7s 137us/sample - loss: 1.2338 - classifier_loss: 0.7603 - embedder_loss: 0.4732 - classifier_accuracy: 0.7344\n",
      "Epoch 4/4\n",
      "49856/50000 [============================>.] - ETA: 0s - loss: 1.1658 - classifier_loss: 0.7212 - embedder_loss: 0.4446 - classifier_accuracy: 0.7456\n",
      "Epoch 00004: saving model to D:/ICG/Experiments/%s/test/classifiermodel-weights-after-embedding-epoch04.hdf5\n",
      "50000/50000 [==============================] - 7s 137us/sample - loss: 1.1653 - classifier_loss: 0.7206 - embedder_loss: 0.4446 - classifier_accuracy: 0.7457\n",
      "saved file to D:/ICG/Experiments/%s/test/classifiermodel-history-after-embedding.json\n",
      "Train on 50000 samples\n",
      "Epoch 1/4\n",
      "49984/50000 [============================>.] - ETA: 0s - loss: 0.8509 - accuracy: 0.7047\n",
      "Epoch 00001: saving model to D:/ICG/Experiments/%s/test/classifiermodel-weights-after-baseline-epoch01.hdf5\n",
      "50000/50000 [==============================] - 7s 131us/sample - loss: 0.8510 - accuracy: 0.7047\n",
      "Epoch 2/4\n",
      "49920/50000 [============================>.] - ETA: 0s - loss: 0.7904 - accuracy: 0.7231\n",
      "Epoch 00002: saving model to D:/ICG/Experiments/%s/test/classifiermodel-weights-after-baseline-epoch02.hdf5\n",
      "50000/50000 [==============================] - 6s 124us/sample - loss: 0.7907 - accuracy: 0.7229\n",
      "Epoch 3/4\n",
      "49856/50000 [============================>.] - ETA: 0s - loss: 0.7584 - accuracy: 0.7353\n",
      "Epoch 00003: saving model to D:/ICG/Experiments/%s/test/classifiermodel-weights-after-baseline-epoch03.hdf5\n",
      "50000/50000 [==============================] - 6s 124us/sample - loss: 0.7578 - accuracy: 0.7355\n",
      "Epoch 4/4\n",
      "49792/50000 [============================>.] - ETA: 0s - loss: 0.7092 - accuracy: 0.7537\n",
      "Epoch 00004: saving model to D:/ICG/Experiments/%s/test/classifiermodel-weights-after-baseline-epoch04.hdf5\n",
      "50000/50000 [==============================] - 6s 122us/sample - loss: 0.7092 - accuracy: 0.7537\n",
      "saved file to D:/ICG/Experiments/%s/test/classifiermodel-history-after-baseline.json\n"
     ]
    }
   ],
   "source": [
    "pli = Latent_Interventions(meta_data, verbose=1)\n",
    "pli()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['D:/ICG/Experiments/%s/test/']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp_path = path+\"/\"\n",
    "from glob import glob\n",
    "import json\n",
    "experiment_paths = glob(exp_path)\n",
    "experiment_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--Warning! Experiment path already exists. By calling further methods, the current files will be overwritten.--\n",
      "Thu Oct 29 10:51:50 2020 Building and compiling search function\n",
      "Pickle of ParametricUMAP model loaded from D:/ICG/Experiments/%s/test/embeddermodel\\model.pkl\n",
      "WARNING:tensorflow:No training configuration found in save file: the model was *not* compiled. Compile it manually.\n",
      "Keras encoder model loaded from D:/ICG/Experiments/%s/test/embeddermodel\\encoder_model.h5\n",
      "WARNING:tensorflow:AutoGraph could not transform <function umap_loss.<locals>.loss at 0x000001C5BB6FB558> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: Cell is empty\n",
      "WARNING: AutoGraph could not transform <function umap_loss.<locals>.loss at 0x000001C5BB6FB558> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: Cell is empty\n",
      "Keras full model loaded from D:/ICG/Experiments/%s/test/embeddermodel\\parametric_model.h5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAx4ElEQVR4nO3deVyVdd7/8dfnsIgsggi4gAiuyA6hYppallo2bVNTpplWpv26p6bGpm1muqd7umvu6Z7bWZrMymymppxKaxrNTC01cwNXRFwRBRdWEVCQ5fv74xxJDRWUwwWcz/PxOI/knOtc15uT8ub6Xtf1vcQYg1JKKddlszqAUkopa2kRKKWUi9MiUEopF6dFoJRSLk6LQCmlXJy71QGaKigoyERERFgdQyml2pT09PRCY0xwQ6+1uSKIiIggLS3N6hhKKdWmiEjOhV7ToSGllHJxWgRKKeXitAiUUsrFtbljBEq1lOrqanJzc6msrLQ6ilKN5uXlRVhYGB4eHo1+jxaBUheQm5uLn58fERERiIjVcZS6JGMMRUVF5ObmEhkZ2ej3OW1oSETmiki+iGRc4HV/EflcRLaKyA4RmeqsLEpdjsrKSrp06aIloNoMEaFLly5N3ot15jGCecC4i7z+KJBpjEkARgH/KyKeTsyjVJNpCai25nL+zjqtCIwxq4Diiy0C+Ik9ta9j2Rpn5cnZtYV1f53G6Sod71VKqbNZedbQX4CBwGFgO/C4MaauoQVF5GERSRORtIKCgsva2PG83aTm/5OMFR9cdmClWpqbmxuJiYkkJCSQnJzMd99916zrnzJlCh9//DEADz30EJmZmVe8zm+++Yabb775itejWo6VRTAW2AL0ABKBv4hIp4YWNMbMMcakGGNSgoMbvEL6kmJH3MFRgnDf+vfLjKtUy+vYsSNbtmxh69atvPzyyzz77LNO29Zbb71FdHS009avWi8ri2AqsMDY7QWygShnbczN3Z3s8DuIr0zncHaWszajlNOcOHGCzp07A1BeXs7o0aNJTk4mLi6Ozz77DICKigrGjx9PQkICsbGxzJ8/H4D09HRGjhzJVVddxdixYzly5MgP1j9q1Kj66Vt8fX15/vnnSUhIIDU1lWPHjgFQUFDAj3/8YwYNGsSgQYNYs2bNRTMXFxdz2223ER8fT2pqKtu2bQNg5cqVJCYmkpiYSFJSEmVlZRw5coQRI0aQmJhIbGwsq1evbp4PTl2SlaePHgRGA6tFpCswANjvzA1G3jCd2rfeJGfZ6/SY9kdnbkq1M7/5fAeZh0806zqje3TihR/FXHSZU6dOkZiYSGVlJUeOHGHFihWA/VzxhQsX0qlTJwoLC0lNTeWWW25hyZIl9OjRg0WLFgFQWlpKdXU1P/3pT/nss88IDg5m/vz5PP/888ydO/eC262oqCA1NZWXXnqJX/ziF7z55pv88pe/5PHHH+eJJ55g+PDhHDx4kLFjx7Jz584LrueFF14gKSmJTz/9lBUrVjB58mS2bNnCq6++ymuvvcawYcMoLy/Hy8uLOXPmMHbsWJ5//nlqa2s5efLkZXyq6nI4rQhE5APsZwMFiUgu8ALgAWCMmQ38FzBPRLYDAjxtjCl0Vh6Abj37ssV7CH3zPqP69P/g4dnBmZtT6oqdGRoCWLt2LZMnTyYjIwNjDM899xyrVq3CZrORl5fHsWPHiIuLY+bMmTz99NPcfPPNXHPNNWRkZJCRkcENN9wAQG1tLd27d7/odj09PevH+a+66iq++uorAJYtW3bOcYQTJ05QVlaGn59fg+v59ttv+eSTTwC47rrrKCoqorS0lGHDhvHkk08yceJE7rjjDsLCwhg0aBAPPPAA1dXV3HbbbSQmJl7JR6eawGlFYIyZcInXDwNjnLX9C0q+n+A1j7D5m49IGjOpxTev2qZL/ebeEoYOHUphYSEFBQUsXryYgoIC0tPT8fDwICIigsrKSvr37096ejqLFy/m2WefZcyYMdx+++3ExMSwdu3aRm/Lw8Oj/jRENzc3amrsJ/TV1dWxdu1aOnbs2Kj1GGN+8JyI8MwzzzB+/HgWL15Mamoqy5YtY8SIEaxatYpFixZx33338dRTTzF58uT6961fv57p06cD8OKLL3LLLbc0+vtRF+dycw3FjrqTfAKxbZpndRSlmiQrK4va2lq6dOlCaWkpISEheHh48PXXX5OTY59h+PDhw3h7ezNp0iRmzpzJpk2bGDBgAAUFBfVFUF1dzY4dOy4rw5gxY/jLX/5S//WZvZULGTFiBO+//z5gP5soKCiITp06sW/fPuLi4nj66adJSUkhKyuLnJwcQkJCmDZtGg8++CCbNm06Z11Dhgxhy5YtbNmyRUugmbncFBPuHp7sD7udwYfmcvTgHrqF97M6klIXdOYYAdh/u3733Xdxc3Nj4sSJ/OhHPyIlJYXExESiouznWWzfvp2nnnoKm82Gh4cHr7/+Op6ennz88cc89thjlJaWUlNTw89+9jNiYpq+l/OnP/2JRx99lPj4eGpqahgxYgSzZ8++4PL/+Z//ydSpU4mPj8fb25t3330XgFmzZvH111/j5uZGdHQ0N954Ix9++CG///3v8fDwwNfXl7/97W9N/8DUZZGGdt1as5SUFHOlN6Y5krOLrnOHsD78QYY++L/NlEy1Nzt37mTgwIFWx1CqyRr6uysi6caYlIaWd7mhIYDuvQaQ0TGFPocWUFN92uo4SillKZcsAoDapPsIoZiMVQusjqKUUpZy2SKIvfYeCgmA9HlWR1FKKUu5bBF4eHZgT49biatYx7HcfVbHUUopy7hsEQCEXz8dNzHs/2qO1VGUUsoyLl0Eob1j2N4hicicT6itcdoM2Eop1aq5dBEAnE64n24UsGP1p1ZHUUopS7h8EcSNnkAxnahNe8fqKEr9wJn7EZx5vPLKK41+75XeF+Bi74+IiKCw0D412NVXX33Z22iMs2dFvVJpaWk89thjAFRVVXH99deTmJjI/Pnzm+1+DGff46GtcLkri8/n2cGL3d1+RMqRDyg8nENQj15WR1Kq3tmTzrVWzX2zHGdKSUkhJcV+TdXmzZuprq6u/3zvvvvuJq2rtrYWNze35o5oCZffIwAIHT0dd6ljz1dvWB1FtVZfPAPvjG/exxfPXHaciIgInnvuOYYOHUpKSgqbNm1i7Nix9OnT55wpH06cOMHtt99OdHQ0M2bMoK7OfhPApUuXMnToUJKTk7nrrrsoLy8HYMmSJURFRTF8+HAWLPj+GpuioiLGjBlDUlIS06dPP2cyOV9fX8C+BzFq1CjuvPNOoqKimDhxYv1yixcvrl/vY4891uCeRm1tLTNnziQuLo74+Hj+/Oc//2CZRx55hJSUFGJiYnjhhRfqn3/mmWeIjo4mPj6emTNnAvDRRx8RGxtLQkICI0aMqM948803k5+fz6RJk9iyZQuJiYns27fvnD2PC30+ERERvPjiiwwfPpyPPvrokv+fli9fTlJSEnFxcTzwwANUVVU1KW9Lcfk9AoCe/RLY4RlPrwMfU1f7X9jaScurtu/suYYAnn322frfXHv27MnatWt54oknmDJlCmvWrKGyspKYmBhmzJgBwIYNG8jMzKRXr16MGzeOBQsWMGrUKH7729+ybNkyfHx8+N3vfscf/vAHfvGLXzBt2jRWrFhB3759z/kN+Te/+Q3Dhw/n17/+NYsWLWLOnIbPtNu8eTM7duygR48eDBs2jDVr1pCSksL06dNZtWoVkZGRTJjQ8MTEc+bMITs7m82bN+Pu7k5x8Q9vef7SSy8RGBhIbW0to0ePZtu2bYSFhbFw4UKysrIQEY4fPw7YZyj98ssvCQ0NrX/ujJCQEN566y1effVV/v3vf5/zWmFhYYOfz69//WvAfi+Ib7/99sL/0xwqKyuZMmUKy5cvp3///kyePJnXX3+dyZMnNzmvs2kROJyKv4+YtKfYvuZfxI243eo4qrW5sfFj883pYkNDZ2bgjIuLo7y8HD8/P/z8/PDy8qr/QTJ48GB69+4NwIQJE/j222/x8vIiMzOTYcOGAXD69GmGDh1KVlYWkZGR9Otnn4hx0qRJ9T/wV61aVb+HMH78+Po7pZ1v8ODBhIWFAZCYmMiBAwfw9fWld+/eREZG1udoqEiWLVvGjBkzcHe3/1gKDAz8wTL//Oc/mTNnDjU1NRw5coTMzEyio6Px8vLioYceYvz48fV7G8OGDWPKlCn85Cc/4Y477rjIp3yudevWNfj5nNHYIaRdu3YRGRlJ//79Abj//vt57bXX+I//+I9mzdscdGjIIXb0RErwo3qDHjRWbUOHDvYbK9lstvo/n/n6zP0DztxT4AwRwRjDDTfcUD+lc2ZmJm+//XaDy5//3sZmgu/vY9DYiS2NMRfdRnZ2Nq+++irLly9n27ZtjB8/nsrKStzd3dmwYQM//vGP+fTTTxk3bhwAs2fP5re//S2HDh0iMTGRoqKiRue40OcD4OPj0+j1NORy8z7//PP1Jw00Ny0CB6+OPuzqOp64sm8pPHrI6jhKNYsNGzaQnZ1NXV0d8+fPZ/jw4aSmprJmzRr27t0LwMmTJ9m9ezdRUVFkZ2ezb5/9SvsPPvigfj1n31fgiy++oKSkpNEZoqKi2L9/PwcOHACov4/y+caMGcPs2bPrS+z8oaETJ07g4+ODv78/x44d44svvgDs928uLS3lpptuYtasWfV7UPv27WPIkCG8+OKLBAUFcehQ4/5dX+jzaaqoqCgOHDhQv56///3vjBw58rLzvvTSS/Xl1Ny0CM7S/drpeEgte5fqlcaqdThzjODM45lnmnaAeejQoTzzzDPExsYSGRnJ7bffTnBwMPPmzWPChAn1N5XPysqqv2/w+PHjGT58OL16fX8G3QsvvMCqVatITk5m6dKlhIeHNzpDx44d+etf/8q4ceMYPnw4Xbt2xd/f/wfLPfTQQ4SHhxMfH09CQgL/+Mc/znk9ISGBpKQkYmJieOCBB+qHbsrKyrj55puJj49n5MiR/N///R8ATz31FHFxccTGxjJixAgSEhIalfdCn09TeXl58c4773DXXXcRFxeHzWZjxowZzZ63Objk/QguJvOlYXSqKSL0V5mITXvSlen9CJpPeXk5vr6+GGN49NFH6devH0888YTVsdotvR/BFaqInUiYOcKOtYusjqJUu/Hmm2+SmJhITEwMpaWl9fceVq2DnjV0nrgbJnNi82+pWjcXhv3I6jhKtQtPPPFEu9sDePTRR1mzZs05zz3++ONMnTrVokSXT4vgPF7evmwNvpGk/E8pKThC5+DuVkdSFrrUmSzKdb322mtWR2jQ5Qz369BQA0JGPYyn1LBLDxq7NC8vL4qKii7rH5ZSVjDGUFRUhJeXV5Pep3sEDYiMGULWwoH02DcfU/crPWjsosLCwsjNzaWgoMDqKEo1mpeXV/1FfY2lRXABJ2LuJWrrr8jcsJTo1HFWx1EW8PDwqL8aVqn2TH/VvYDYG+6nzHTk5HdvWR1FKaWcSovgArx9/ckMvpG40m8oLTpmdRyllHIapxWBiMwVkXwRybjIMqNEZIuI7BCRlc7KcrmCRkyjg1Szc6nuFSil2i9n7hHMAy44uC4iAcBfgVuMMTHAXU7Mcln6xF/Nbvf+dNvzIcYxj7tSSrU3TisCY8wq4IcTin/vXmCBMeagY/l8Z2W5EscH3ktE3UF2pa+wOopSSjmFlccI+gOdReQbEUkXkckXWlBEHhaRNBFJa+lT+WLGTKXCeFG2RoeHlFLtk5VF4A5cBYwHxgK/EpH+DS1ojJljjEkxxqQEBwe3ZEZ8/ALI6DKG2JLllJYUtui2lVKqJVhZBLnAEmNMhTGmEFgFtNy8q00QOOJhOsppspa+femFlVKqjbGyCD4DrhERdxHxBoYAOy3Mc0H9Eq9hr1sfgnf9Qw8aK6XaHWeePvoBsBYYICK5IvKgiMwQkRkAxpidwBJgG7ABeMsYc8FTTa1WNGACvesOsGfLKqujKKVUs3LaFBPGmAmNWOb3wO+dlaE5RY99kJM7fs/x1W9C8iir4yilVLPRK4sbyc8/kIzOo4kt/oqy0oudFauUUm2LFkET+A+fhrdUkbl0rtVRlFKq2WgRNEH/5FFk2yIIzPrA6ihKKdVstAiaQGw28vvfQ7/avezZstrqOEop1Sy0CJooauw0Ko0HxavftDqKUko1Cy2CJvLvHMT2gOuIKVxKRdlxq+MopdQV0yK4DH5XP4SvnGLH0nlWR1FKqSumRXAZBgy6ngO2ngTsfN/qKEopdcW0CC6D2Gwc7Xs3/Wt2s2/7OqvjKKXUFdEiuExRY6ZRZTwoXPmG1VGUUuqKaBFcpoCgbmz3H8nAwiWcqiizOo5SSl02LYIr0DH1QTpxkoyv3rU6ilJKXTYtgisQnTqOQ9IDvx160Fgp1XZpEVwBsdnI63M3UdWZHNiZZnUcpZS6LFoEV6j/mGmcNm4c/VoPGiul2iYtgisUGBLK9k4jGJi/iMpTFVbHUUqpJtMiaAYdBk/Fnwoyvvq71VGUUqrJtAiaQfTVN5Mr3fDOeM/qKEop1WRaBM3A5ubGocifEH16Ozm7tlgdRymlmkSLoJn0G/Mw1caNIytmWx1FKaWaRIugmQR168l232EMOPZvqipPWh1HKaUaTYugGXkMnkpnyti+XC8wU0q1HVoEzShm+K0clhC8tulBY6VU26FF0Ixsbm7k9PoxsVVbOLR3u9VxlFKqUbQImlnfMTOoMTZyl+tBY6VU26BF0MyCe0Sw3SeV/kc+53RVpdVxlFLqkrQInMCWMpUulJKx4gOroyil1CU5rQhEZK6I5ItIxiWWGyQitSJyp7OytLTYEXdwlCDct+qUE0qp1s+ZewTzgHEXW0BE3IDfAV86MUeLc3N3Jzv8DuIr0zmcnWV1HKWUuiinFYExZhVQfInFfgp8AuQ7K4dVIm+YTq0Rcpa9bnUUpZS6KMuOEYhIKHA70C5Pr+nWsy/bvYfQN+8zqk9XWR1HKaUuyMqDxbOAp40xtZdaUEQeFpE0EUkrKChwfrLmknw/wZSQ8c1HVidRSqkLsrIIUoAPReQAcCfwVxG5raEFjTFzjDEpxpiU4ODgFox4ZWJH3Uk+gdg2zbM6ilJKXZBlRWCMiTTGRBhjIoCPgf9njPnUqjzO4O7hyb6w24g7lcbRg3usjqOUUg1y5umjHwBrgQEikisiD4rIDBGZ4axttka9rrd/u9lftctDIUqpdsDdWSs2xkxowrJTnJXDaj0iBrCtYwp9Di2gpvpl3D08rY6klFLn0CuLW0Bt0n2EUEzGqgVWR1FKqR/QImgBsdfeQyEBkD7P6ihKKfUDWgQtwMOzA3t63EpcxTqO5e6zOo5SSp1Di6CFhF8/HTcx7P9qjtVRlFLqHFoELSS0dwzbOyQRmfMJtTU1VsdRSql6WgQt6HTC/XSjgIzVC62OopRS9bQIWlDc6AkU04m6tHlWR1FKqXpaBC3Is4MXu7v9iLjy7yg8nGN1HKWUArQIWlzo6Om4Sx17l75hdRSllAK0CFpcz34J7PCMJzznY+pqLznxqlJKOZ0WgQVOxd9HD3OMHWv+ZXUUpZTSIrBC7OiJHMeX6g3vWB1FKaW0CKzg1dGHrK43E1f2LYVHD1kdRynl4rQILNL92ul4SC17l+qVxkopa2kRWKRXVDI7PWIIy/4IU1dndRyllAtrVBGIyOMi0kns3haRTSIyxtnh2rvy2EmEmSPsWLvI6ihKKRfW2D2CB4wxJ4AxQDAwFXjFaalcRNwNkzmBD1Xr5lodRSnlwhpbBOL4703AO8aYrWc9py6Tl7cvO4NvJO7EKg7t3W51HKWUi2psEaSLyFLsRfCliPgBOrDdDELHPsFJ8cL/vbFsX6l3MFNKtbzGFsGDwDPAIGPMScAD+/CQukJhfWOpmLyMIlsw0SseYN17L+jBY6VUi2psEQwFdhljjovIJOCXQKnzYrmW0N4DCXliJVv9riF17yzSZ93FqYoyq2MppVxEY4vgdeCkiCQAvwBygL85LZUL8vELIOnJz1gb8QjJpcvJ+8NIjh7cY3UspZQLaGwR1BhjDHAr8EdjzB8BP+fFck1iszF0yitsGzGbrjWH8Zx7HZlrv7A6llKqnWtsEZSJyLPAfcAiEXHDfpxAOUHi6HsovncJ5TY/+i2ZyPr5r+hxA6WU0zS2CO4GqrBfT3AUCAV+77RUil4DEgl4bDWZ3ikM2fkyG/88iarKk1bHUkq1Q40qAscP//cBfxG5Gag0xugxAifrFNCF2J8vZm3oVAaXLCL71ev0zmZKqWbX2CkmfgJsAO4CfgKsF5E7nRlM2bm5uzN02iw2DZlFePV+6uaMYlfaCqtjKaXakcYODT2P/RqC+40xk4HBwK+cF0udL/nGqRy963NqxIPIz+9iw8I/Wx1JKdVONLYIbMaY/LO+LrrUe0Vkrojki0jGBV6fKCLbHI/vHKemqovoHTsE70dXsdsrlsFbf8m61x6i+nSV1bGUUm1cY4tgiYh8KSJTRGQKsAhYfIn3zAPGXeT1bGCkMSYe+C9AJ+ZvhICgbkTN/Ip1Xe8hteAjdr96AyUFR6yOpZRqwxp7sPgp7D+o44EEYI4x5ulLvGcVUHyR178zxpQ4vlwHhDUqscLdw5PUR95gY+J/07cqk8rXrmHftu+sjqWUaqMafWMaY8wnxpgnjTFPGGMWNnOOB4ELXjklIg+LSJqIpBUUFDTzptuuQbc9ysHbFmCjjh6f3Er6oresjqSUaoMuNc5fJiInGniUiciJ5gggItdiL4IL7mEYY+YYY1KMMSnBwcHNsdl2o1/SCNxmfMMBz75ctfHnrH3jp9TW1FgdSynVhly0CIwxfsaYTg08/Iwxna504yISD7wF3GqMKbrS9bmqoG7h9Jn5NesDb2Hokb+R8eo4SksKrY6llGojLLtnsYiEAwuA+4wxu63K0V54dvBiyGN/Z33Mr4g+tYkTf7qGnJ3pVsdSSrUBTisCEfkAWAsMEJFcEXlQRGaIyAzHIr8GugB/FZEtIpLmrCyuZMhdM9l70wd4mwq6fDiezUvfszqSUqqVE/ukom1HSkqKSUvTzriUo4f2UjbvbvrV7mVt+HSG3P8yNjc3q2MppSwiIunGmJSGXrNsaEg5V7eefen585Vs9B/D0INvsPUPt1B+ouTSb1RKuRwtgnbMy9uXlMfns67fz4kvX0PhrBHk7m3wQm+llAvTImjnxGYjdeKvyRz9LgF1xXR6bwzbVy6wOpZSqhXRInARcSNupWLyMopswUSveIB1772gN7tRSgFaBC4ltPdAQp5YyVa/a0jdO4v0WXdxqqLM6lhKKYtpEbgYH78Akp78jLURj5Bcupy8P4zk6ME9VsdSSllIi8AFic3G0CmvsG3EbLrWHMZz7nVkrr3gVE9KqXZOi8CFJY6+h+J7l1Bu86Pfkomsn/+KHjdQygVpEbi4XgMSCXhsNZneKQzZ+TIb/zyJqsqTVsdSSrUgLQJFp4AuxP58MWtDpzK4ZBHZr15H4eEcq2MppVqIFoECwM3dnaHTZrFpyCzCq/dTN2cUu9JWWB1LKdUCtAjUOZJvnMrRuz6nRjyI/Pwu1r75OGWlF7zRnFKqHdAiUD/QO3YI3o+uYpv/KIbmzeP0/yWy/p+/p6b6tNXRlFJOoEWgGhQQ1I2UJz9h9y3/4phHT4Zk/pa8l5PZuuJDPbNIqXZGi0BdVP/kkQx8djWbr34Nm6klYdV0dvzuWvZt+87qaEqpZqJFoC5JbDaSxkyi6zObWTfgaUKr9hH5yU1snHUP+XnZVsdTSl0hLQLVaJ4dvEid8By2x7ewofu9JJR8hd+cwax760m914FSbZgWgWoy/85BpM74K4VT1pDZaTipuW9T+YdENnz8Bz2grFQbpEWgLluPyCiu+vlCdt28kAKPHgzO+A25L1/Ftq8/tjqaUqoJtAjUFRuQch1Rz65hU+ofcTeniV/5INteuY79GeutjqaUagQtAtUsxGYjedwUQp7Zyrr+M+lVmUXER2PZMGsCBYcPWB1PKXURWgSqWXl28CL13l/BY1vZ0O0eEku+xOeNwax9eyYny0utjqeUaoAWgXIK/8BgUh+ZTcH9q8nyS2XooTepeDWBDZ/Moramxup4SqmzaBEopwrtHUPyzH+RddPHFLt3ZfD2F8j576vYvmqh1dGUUg5aBKpFRA2+gf7PrSV98Cy8zCniVkxh2yvXk5250epoSrk8LQLVYsRm46qbptLl6S2s6/ckEZWZhM+/gQ1/mkTh0YNWx1PKZWkRqBbXwcub1IkvUPcfm9jY9SckFi3G+/UU1r7zNKcqyqyOp5TLcVoRiMhcEckXkYwLvC4i8icR2Ssi20Qk2VlZVOsUENSN1P83h/z7VpHlO4ShObMp+308Gxb+WQ8oK9WCnLlHMA8Yd5HXbwT6OR4PA687MYtqxcL6xpL81OfsvPGfHHcPYvDWX3Lg5UFkrP7M6mhKuQSnFYExZhVwsVtb3Qr8zditAwJEpLuz8qjWb+CQsfR7bj1pg17Fu7aM2OWT2fq7MeRkbbI6mlLtmpXHCEKBQ2d9net47gdE5GERSRORtIKCghYJp6whNhsp46fR+emtrOvzOJGnthP6wWjW//l+Co8euvQKlFJNZmURSAPPmYYWNMbMMcakGGNSgoODnRxLtQZeHX1Ive9Fah/dRHrIHSQXfo7X64NYN+85Kk+WWx1PqXbFyiLIBXqe9XUYcNiiLKqV6hzcnSGPvs3RSV+zxyeZ1AOvUfo/8az7268ozs+zOp5S7YKVRfAvYLLj7KFUoNQYc8TCPKoV69kvgaRfLCZz7IcUefYgdf+f8H0tjvT/vY2MNZ/rfZSVugJiTIOjMVe+YpEPgFFAEHAMeAHwADDGzBYRAf6C/cyik8BUY0zapdabkpJi0tIuuZhq53J2pnPk69lE5y+iExUckh7k9b2HqLHTCQjqZnU8pVodEUk3xqQ0+JqzisBZtAjU2SpPlrN96Tz8Mt4jqmYnVcaD7f4j8b56GgMHj0Fses2kUqBFoFxE9o715H89m+iCL/CTU+TYenKk7z0MHDcd/0A9yUC5Ni0C5VJOlpeyY+k8/DPfo3/NbiqNB9sDrsNv2DQGpIzWvQTlkrQIlMvat+07Cle+QWzhEnykkmxbL/L7TyBq7DT8OwdZHU+pFqNFoFxeRdlxdnw5l84736df7V5OGU+2d74e/+HT6J88SvcSVLunRaDUWfZsXkXJqjeILf4Kb6lin1skhQPuJXrsQ/j5B1odTymn0CJQqgFlpcVkfvkWQbv+QZ/abE6aDmR0uYHAa6bTN2mE1fGUalZaBEpdhKmrY/embyj99k3iSpbRUU6z160PxQMnEjP2QXz8AqyOqNQV0yJQqpFKSwrJ+vJNQnZ/QGRdDhXGix1BY+kycjp94odZHU+py6ZFoFQTmbo6dqUtp2zNm8QdX4GXVLPbvT+l0ZOIGTMFb19/qyMq1SRaBEpdgdKiY+z88k267/2QXnWHKDMdyQy+kZBrZxAZM8TqeEo1ihaBUs3A1NWxc8NSTn73JnGlK+kg1WS5D6QsdhJxY6bg5e1rdUSlLkiLQKlmdrzwKFlfvkHo3g/paQ5zAh8yg2/CM3kC/RKvwa+jp9URlTqHFoFSTmLq6shc+wWV694i7sRKPKWWQyaY9V7XUNzrRrpHD2NQZBe6+XtZHVW5OC0CpVpA+fECDq/7GI9d/6JnyXrcqSXXBLG4dghpPiPwjhhMSmQXBkUE0i/EF5utoZv0KeUcWgRKtbRTJdTsXMTJzR/jk7saN1PDEYL4d81gFtWmsr/DAFIiunBVr84MiggkPswfLw83q1OrdkyLQCkrnSqBXV9gdnwK+1YgddWUeHRlmaTyj7JkNpu+eLq5ERfmT0pEZ1J6BZLSqzOdffQ4g2o+WgRKtRanjsOuxeAoBeqqOeXdne1+I1lwehAL8rtxutY+ZNQ3xJdBZ4ohojPhgd7Yb+ynVNNpESjVGp06Dru+gMxP7aVQexrj14NjYeNY02E4i4pD2XiwlLLKGgCC/TqcUwzR3Tvh7qazpqrG0SJQqrWrLLWXwo5PYd9yqD0NnUIxA3/Ewe7jWH0qgrSc42w8UELe8VMAeHu6kRQeQEqvQAZFBJIUHoBPB3drvw/VamkRKNWWVJbCriX2PYW9y+yl4NcDom+FmNs40imOtJxS0g4Us/FACTuPnsAYcLMJA7v71RdDSkRnunbS01aVnRaBUm1V5QnYvcS+p7B3GdRWOUrhFoi+DXoO4cTpWjYfPE66oxg2HyqhsroOgPBAb1IiOjO8bxAj+wfTxbeDpd+Oso4WgVLtQYOl0B0G3gIxt0HPVLDZqK6tY8fhE6QdKCbtQAkbDxRTVHEaEYgPC2BU/2CujQohPtRfr2VwIVoESrU3lSdg95f24aM9X9lLwbebfU8h5vb6UgCoqzNkHC7lm10FfL0rny2HjmMMdPHxZGT/YEYOCGZEv2A9XbWd0yJQqj2rKrOXwo6F9j2FmsrvSyH6NghPBdv3F6sVV5xm9Z4Cvs7KZ+XuAkpOVmMTSArvXL+3EN29k+4ttDNaBEq5ijOlcGZPoaYSfLtCr2EQlgKhKdA9Hjw6AlBbZ9iWe5yvdxWwclc+W3NLAfupqiP7B3PtgBCG9wvCv6OHhd+Uag5aBEq5oqpy+zGFrEWQuxFKD9mft7lD1xh7KZwphy59wWajoKyKVbvtQ0ir9xRSeqoaN5twVXhnRkXZiyGqm59e2NYGaREopaDsKOSlQ24a5KVB3mY4XWZ/rYM/hCadUw41Hbuw5dBxvt6Vzze7Cthx+AQA3Tp5MWpAMKMGhDCsbxf8vHRvoS2wrAhEZBzwR8ANeMsY88p5r/sD7wHhgDvwqjHmnYutU4tAqWZSVweFu+2lkJtmL4ljO8DU2l/3D4ewq+rLId+nP99kV/DNrnxW7y6krKoGd5swKCKQa6PsxdAvxFf3FlopS4pARNyA3cANQC6wEZhgjMk8a5nnAH9jzNMiEgzsAroZY05faL1aBEo50emTcGSrY48hHXLTofSg/TVxsw8phaVQ0+MqdtCPL4768c3uQrKO2vcsQgM61u8tXN2ni17p3IpcrAic+X9pMLDXGLPfEeJD4FYg86xlDOAn9l8hfIFioMaJmZRSF+PpDb2G2h9nlB2zl8KZctj+Me5pc0kAEjp04pkeSZT3T2RTbR/+VdCdTzfn8f76g3i62RjSO9B+0DkqhN5BPrq30Eo5c4/gTmCcMeYhx9f3AUOMMf9x1jJ+wL+AKMAPuNsYs6iBdT0MPAwQHh5+VU5OjlMyK6Uaoa4OivZ8P5yUl2YfUqqz/w5nOoVRFBDPFtOHfxeFsqS4K5V0IDzQm1ED7AecU3t3oaOn3n+hJVm1R9BQ9Z/fOmOBLcB1QB/gKxFZbYw5cc6bjJkDzAH70FDzR1VKNZrNBsED7I+kifbnqk/Zh5Ry05C8dILy0rj++GKuB0xHN0p8+7GNvixND+Oldb3JdQtjSO9g4sP86RviS59gXyKDfHQoySLO/NRzgZ5nfR0GHD5vmanAK8a+W7JXRLKx7x1scGIupVRz8+hov3AtPPX758rzIS8dyUsnMDeNUXmrGWUrhQ5QZfNmR15fdmaHkFUXxHITRJ7pQo1fKP4hPekd4k+fYB/6hPjSN9iXYL8OOqzkRM4sgo1APxGJBPKAe4B7z1vmIDAaWC0iXYEBwH4nZlJKtRTfEBhwo/0BjiGlvZCXToe8NJLzNpFUko6cKvn+PaehJteNI4e6kGe6kGeC2GiCKHLrCgFhdAyKoHOP3kR07ULfEB96dfHBQ+/JcMWcffroTcAs7KePzjXGvCQiMwCMMbNFpAcwD+iOfSjpFWPMexdbp541pFQ7U1UOpbn2C96OH4TSXEzpIaqLcjDHD+JxMh8bdee8pcD4k2e6cIRgyjp0p7ZTGJ6B4fh1603X8H5EhPbA31vnTjqbXlCmlGq7aqvhxGFHURyiqjiH8mPZ1BQfxL0sD7+qI3ied8Z5ufHimART2qEb1b6huHUOxzskkqDQPgSF9sHWqfs58y+5AqsOFiul1JVz84DOvewPoIPjUc8YqCikpiSHorz9lB7ZR1VRDlJ6iIBThwks2klAUTns/f4t1bhT6hHMKe8e4B+GV1AEAd174xHYCwLCIaAXuLnOj0fX+U6VUu2TCPgG4+4bTNeeKXQ972VjDEUlxeQd2E3R4f2czM/GHD9Eh4o8AkqOEXp8NYE5/8JNvh8dqRFPTnUeQIeeiXiGJkD3BAiJhg6+Lfu9tRAdGlJKuaxTp2vJLqxg/7ESjuXup+xYNrVFB+hUtpdoOUCM7QABUgGAQajt3Bv3HgnQLc4+i2u3ePtB8TZAjxEopVQTlFVWs+XQcTZmF3Ng/y5qD2+jT202MbYDxLkdpAf59csa325IfTHE2cuhc2T9jYFaCy0CpZS6AjW1dew8UkZaTjFpOSXsyj5IUMUeYuQA8e4HSfI8RFj1QWw4Juzz9IWuseeWQ8hAcLfuntFaBEop1YyMMeSWnCI9x35P6PScErKPFdGXPOLcchjue5h4t4P0qNqLe81J+5ts7hAc9X0xdIuDbrHQsXOLZNYiUEopJys9Vc2mgyWkH7CXw9bc41RV19BLjnGN3xFG+B0l2naArhV7cD957Ps3BoQ7iiH++2MPnULtB8GbkZ4+qpRSTubf0YNrB4Rw7QD7wePTNXXsOFxKek4JaQdKeDanmMJy+/UOEV7l/CikkKE+RxhgsulcsAtb1iLqp2PrGOjYY3DsPXSPhy79nHZKq+4RKKVUCzDGkFN0sn4oKS2nhL355QB4uAkp3T0ZG1zEYK9cetdm41WYAccyobbKvgJ3L7hmJox86rK2r3sESillMREhIsiHiCAf7kqxz8dZUnG6vhTSDhTz39uqOV0bBUQRGXQngwf4MSqolGTPQ4RU7Ea6xjglmxaBUkpZpLOPJ9dHd+X6aPtlcJXVtWTklTqKoYSlWUXMP1kNBBHo04NHOvZhWlTz59AiUEqpVsLLw42UiEBSIgJhpH04aV9BBek5xWw8UEJXfy+nbFeLQCmlWikRoW+IL31DfLl7ULjTttO6Ln1TSinV4rQIlFLKxWkRKKWUi9MiUEopF6dFoJRSLk6LQCmlXJwWgVJKuTgtAqWUcnFtbtI5ESkAci7z7UFAYTPGaS6tNRe03myaq2k0V9O0x1y9jDHBDb3Q5orgSohI2oVm37NSa80FrTeb5moazdU0rpZLh4aUUsrFaREopZSLc7UimGN1gAtorbmg9WbTXE2juZrGpXK51DECpZRSP+RqewRKKaXOo0WglFIuziWKQETmiki+iGRYneVsItJTRL4WkZ0iskNEHrc6E4CIeInIBhHZ6sj1G6sznU1E3ERks4j82+osZ4jIARHZLiJbRCTN6jxniEiAiHwsIlmOv2dDW0GmAY7P6czjhIj8zOpcACLyhOPvfIaIfCAizrklWBOJyOOOTDuc8Vm5xDECERkBlAN/M8bEWp3nDBHpDnQ3xmwSET8gHbjNGJNpcS4BfIwx5SLiAXwLPG6MWWdlrjNE5EkgBehkjLnZ6jxgLwIgxRjTqi5CEpF3gdXGmLdExBPwNsYctzhWPRFxA/KAIcaYy71QtLmyhGL/ux5tjDklIv8EFhtj5lmcKxb4EBgMnAaWAI8YY/Y01zZcYo/AGLMKKLY6x/mMMUeMMZscfy4DdgKh1qYCY1fu+NLD8WgVvzGISBgwHnjL6iytnYh0AkYAbwMYY063phJwGA3ss7oEzuIOdBQRd8AbOGxxHoCBwDpjzEljTA2wEri9OTfgEkXQFohIBJAErLc4ClA//LIFyAe+Msa0ilzALOAXQJ3FOc5ngKUiki4iD1sdxqE3UAC84xhKe0tEfKwOdZ57gA+sDgFgjMkDXgUOAkeAUmPMUmtTAZABjBCRLiLiDdwE9GzODWgRtAIi4gt8AvzMGHPC6jwAxphaY0wiEAYMduyeWkpEbgbyjTHpVmdpwDBjTDJwI/CoYzjSau5AMvC6MSYJqACesTbS9xxDVbcAH1mdBUBEOgO3ApFAD8BHRCZZmwqMMTuB3wFfYR8W2grUNOc2tAgs5hiD/wR43xizwOo853MMJXwDjLM2CQDDgFsc4/EfAteJyHvWRrIzxhx2/DcfWIh9PNdquUDuWXtzH2MvhtbiRmCTMeaY1UEcrgeyjTEFxphqYAFwtcWZADDGvG2MSTbGjMA+zN1sxwdAi8BSjoOybwM7jTF/sDrPGSISLCIBjj93xP4PJMvSUIAx5lljTJgxJgL7kMIKY4zlv7GJiI/jYD+OoZcx2HfnLWWMOQocEpEBjqdGA5aeiHCeCbSSYSGHg0CqiHg7/m2Oxn7cznIiEuL4bzhwB838ubk358paKxH5ABgFBIlILvCCMeZta1MB9t9w7wO2O8bjAZ4zxiy2LhIA3YF3HWd02IB/GmNazamarVBXYKH9ZwfuwD+MMUusjVTvp8D7jmGY/cBUi/MA4BjrvgGYbnWWM4wx60XkY2AT9qGXzbSeqSY+EZEuQDXwqDGmpDlX7hKnjyqllLowHRpSSikXp0WglFIuTotAKaVcnBaBUkq5OC0CpZRycVoESrUgERnVmmZNVQq0CJRSyuVpESjVABGZ5LgnwxYRecMxCV+5iPyviGwSkeUiEuxYNlFE1onINhFZ6JizBhHpKyLLHPd12CQifRyr9z3rHgHvO65iVcoyWgRKnUdEBgJ3Y59ILhGoBSYCPtjnxknGPhXwC463/A142hgTD2w/6/n3gdeMMQnY56w54ng+CfgZEI19htBhTv6WlLool5hiQqkmGg1cBWx0/LLeEft03HXAfMcy7wELRMQfCDDGrHQ8/y7wkWPuoVBjzEIAY0wlgGN9G4wxuY6vtwAR2G+IopQltAiU+iEB3jXGPHvOkyK/Om+5i83PcrHhnqqz/lyL/jtUFtOhIaV+aDlw51kzPgaKSC/s/17udCxzL/CtMaYUKBGRaxzP3wesdNxXIldEbnOso4NjojWlWh39TUSp8xhjMkXkl9jvOGbDMeMj9hu7xIhIOlCK/TgCwP3AbMcP+rNn+LwPeENEXnSs464W/DaUajSdfVSpRhKRcmOMr9U5lGpuOjSklFIuTvcIlFLKxekegVJKuTgtAqWUcnFaBEop5eK0CJRSysVpESillIv7//woy/g8c6jmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fit_intervention.plot_histories(experiment_paths, metric_key_base=\"loss\", metric_key_embed=\"classifier_loss\", plot_baseline=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}