results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import altair as alt\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_df_from_history(filepath, method_name):\n",
    "    with open(filepath, \"r\") as f:\n",
    "        result = json.load(f)\n",
    "    result_df = pd.DataFrame(result)\n",
    "    result_df[\"epoch\"] = result_df.index.values + 1\n",
    "    result_df[\"method\"] = method_name\n",
    "    return result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_dfs = []\n",
    "\n",
    "method_corr = {\n",
    "    \"kpm_nto\": \"KPM\",\n",
    "    \"kpm\": \"KPM*\",\n",
    "    \"yake\": \"YAKE*\",\n",
    "    \"positionrank\": \"PositionRank*\",\n",
    "    \"dev\": \"Manual\"\n",
    "}\n",
    "\n",
    "for method in [\"kpm\", \"yake\", \"positionrank\", \"kpm_nto\", \"dev\"]:\n",
    "    result_df = get_df_from_history(f\"results/spe18_20e_history/{method}.json\", method_corr[method])\n",
    "    result_dfs.append(result_df)\n",
    "    \n",
    "results_df = pd.concat(result_dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['val_loss', 'val_crf_ner_loss', 'val_crf_term_loss',\n",
       "       'val_cls_rel_loss', 'loss', 'crf_ner_loss', 'crf_term_loss',\n",
       "       'cls_rel_loss', 'f1_ner', 'f1_term', 'f1_rel', 'epoch', 'method'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-3d8f42aa10c34c20a128346dc6a2f27e\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-3d8f42aa10c34c20a128346dc6a2f27e\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-3d8f42aa10c34c20a128346dc6a2f27e\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}, \"axis\": {\"labelFont\": \"Times New Roman\", \"labelFontSize\": 18, \"titleFont\": \"Times New Roman\", \"titleFontSize\": 18}, \"legend\": {\"labelFont\": \"Times New Roman\", \"labelFontSize\": 18, \"titleFont\": \"Times New Roman\", \"titleFontSize\": 20}}, \"hconcat\": [{\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"Training Set\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"val_crf_ner_loss\", \"scale\": {\"domain\": [9, 10.4]}, \"title\": \"NER Loss\"}}}, {\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"Training Set\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"val_crf_term_loss\", \"scale\": {\"domain\": [4, 5]}, \"title\": \"EE Loss\"}}}, {\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"Training Set\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"val_cls_rel_loss\", \"scale\": {\"domain\": [0, 0.6]}, \"title\": \"RE Loss\"}}}], \"data\": {\"name\": \"data-1e9ecb5697c5ac9bc28637dbf161d151\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-1e9ecb5697c5ac9bc28637dbf161d151\": [{\"val_loss\": 15.301897678932134, \"val_crf_ner_loss\": 10.182450364105893, \"val_crf_term_loss\": 4.664121624327054, \"val_cls_rel_loss\": 0.4553257749028449, \"loss\": 13.774118158552382, \"crf_ner_loss\": 9.44224182764689, \"crf_term_loss\": 3.892859763569302, \"cls_rel_loss\": 0.43901652925544316, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"With entity only\"}, {\"val_loss\": 15.088710158410734, \"val_crf_ner_loss\": 10.042704025324245, \"val_crf_term_loss\": 4.624338203973144, \"val_cls_rel_loss\": 0.4216679147560231, \"loss\": 14.391434563530815, \"crf_ner_loss\": 9.721958584255642, \"crf_term_loss\": 4.309103541904026, \"cls_rel_loss\": 0.3603723694880803, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 2, \"method\": \"With entity only\"}, {\"val_loss\": 14.868212205650163, \"val_crf_ner_loss\": 9.94457231994963, \"val_crf_term_loss\": 4.530394581112549, \"val_cls_rel_loss\": 0.3932453250362925, \"loss\": 12.414401796129015, \"crf_ner_loss\": 8.338009569380018, \"crf_term_loss\": 3.7816480265723333, \"cls_rel_loss\": 0.29474418196413255, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.021929824561403508, \"epoch\": 3, \"method\": \"With entity only\"}, {\"val_loss\": 14.644684871617894, \"val_crf_ner_loss\": 9.838034796888811, \"val_crf_term_loss\": 4.464446843975652, \"val_cls_rel_loss\": 0.3422032877476546, \"loss\": 12.452283435397678, \"crf_ner_loss\": 8.426005283991495, \"crf_term_loss\": 3.7280285490883722, \"cls_rel_loss\": 0.29824961970249814, \"f1_ner\": 0.0, \"f1_term\": 0.012371134020618556, \"f1_rel\": 0.33200531208499334, \"epoch\": 4, \"method\": \"With entity only\"}, {\"val_loss\": 14.48356381646038, \"val_crf_ner_loss\": 9.733082833951407, \"val_crf_term_loss\": 4.437290826853174, \"val_cls_rel_loss\": 0.3131902344035406, \"loss\": 12.599537902408176, \"crf_ner_loss\": 8.55139226383633, \"crf_term_loss\": 3.790961265563965, \"cls_rel_loss\": 0.2571844566199515, \"f1_ner\": 0.0, \"f1_term\": 0.031862745098039214, \"f1_rel\": 0.33602347762289064, \"epoch\": 5, \"method\": \"With entity only\"}, {\"val_loss\": 14.285567158330096, \"val_crf_ner_loss\": 9.62895664333427, \"val_crf_term_loss\": 4.376269838235674, \"val_cls_rel_loss\": 0.28034074550127464, \"loss\": 11.80238119761149, \"crf_ner_loss\": 8.021390702989367, \"crf_term_loss\": 3.538855208290948, \"cls_rel_loss\": 0.24213523831632403, \"f1_ner\": 0.03489640130861505, \"f1_term\": 0.1641949152542373, \"f1_rel\": 0.4579710144927536, \"epoch\": 6, \"method\": \"With entity only\"}, {\"val_loss\": 14.190792644110909, \"val_crf_ner_loss\": 9.552197621686615, \"val_crf_term_loss\": 4.374607987647509, \"val_cls_rel_loss\": 0.2639873599919089, \"loss\": 15.422736909654406, \"crf_ner_loss\": 10.54382954703437, \"crf_term_loss\": 4.651935312483046, \"cls_rel_loss\": 0.22697221653329003, \"f1_ner\": 0.2111237230419977, \"f1_term\": 0.24483917426788285, \"f1_rel\": 0.4822411205602801, \"epoch\": 7, \"method\": \"With entity only\"}, {\"val_loss\": 14.100167079563558, \"val_crf_ner_loss\": 9.497426730872942, \"val_crf_term_loss\": 4.346293905355634, \"val_cls_rel_loss\": 0.2564464663937144, \"loss\": 13.520169470045301, \"crf_ner_loss\": 9.212770409054226, \"crf_term_loss\": 4.079206003083123, \"cls_rel_loss\": 0.2281930512852139, \"f1_ner\": 0.2951269732326699, \"f1_term\": 0.34651447207501024, \"f1_rel\": 0.48416112631990615, \"epoch\": 8, \"method\": \"With entity only\"}, {\"val_loss\": 14.051934238767972, \"val_crf_ner_loss\": 9.458889895111975, \"val_crf_term_loss\": 4.336595517875504, \"val_cls_rel_loss\": 0.25644875138345424, \"loss\": 12.486466672685411, \"crf_ner_loss\": 8.513386329015097, \"crf_term_loss\": 3.770048565334744, \"cls_rel_loss\": 0.20303168065018123, \"f1_ner\": 0.34801624492346145, \"f1_term\": 0.3886328725038403, \"f1_rel\": 0.5049739459971578, \"epoch\": 9, \"method\": \"With entity only\"}, {\"val_loss\": 14.029026644073264, \"val_crf_ner_loss\": 9.425868076129552, \"val_crf_term_loss\": 4.339041190425845, \"val_cls_rel_loss\": 0.2641175667734912, \"loss\": 11.872677952105645, \"crf_ner_loss\": 8.10128795533991, \"crf_term_loss\": 3.5802190727681955, \"cls_rel_loss\": 0.19117092780455594, \"f1_ner\": 0.4203262233375157, \"f1_term\": 0.4071246819338422, \"f1_rel\": 0.4720161834120027, \"epoch\": 10, \"method\": \"With entity only\"}, {\"val_loss\": 13.972422105552507, \"val_crf_ner_loss\": 9.398135232229302, \"val_crf_term_loss\": 4.321064592277916, \"val_cls_rel_loss\": 0.2532223985142951, \"loss\": 12.673591242896187, \"crf_ner_loss\": 8.674006409115261, \"crf_term_loss\": 3.8085707426071167, \"cls_rel_loss\": 0.19101394216219583, \"f1_ner\": 0.4384251968503937, \"f1_term\": 0.454308093994778, \"f1_rel\": 0.5206316861946001, \"epoch\": 11, \"method\": \"With entity only\"}, {\"val_loss\": 13.9664681636504, \"val_crf_ner_loss\": 9.375703832528886, \"val_crf_term_loss\": 4.348035911573981, \"val_cls_rel_loss\": 0.2427283042538775, \"loss\": 11.879486507839626, \"crf_ner_loss\": 8.127701388465034, \"crf_term_loss\": 3.56903256310357, \"cls_rel_loss\": 0.18275263408819833, \"f1_ner\": 0.41350210970464135, \"f1_term\": 0.44039603960396045, \"f1_rel\": 0.5142371440713982, \"epoch\": 12, \"method\": \"With entity only\"}, {\"val_loss\": 13.93406541504129, \"val_crf_ner_loss\": 9.349364701848831, \"val_crf_term_loss\": 4.344773184644045, \"val_cls_rel_loss\": 0.23992731427624278, \"loss\": 12.419934537675646, \"crf_ner_loss\": 8.50323814815945, \"crf_term_loss\": 3.7242375877168445, \"cls_rel_loss\": 0.19245871818727917, \"f1_ner\": 0.44039530574428665, \"f1_term\": 0.4196141479099679, \"f1_rel\": 0.5310245310245311, \"epoch\": 13, \"method\": \"With entity only\"}, {\"val_loss\": 13.889922232523451, \"val_crf_ner_loss\": 9.325161039394184, \"val_crf_term_loss\": 4.326426059660251, \"val_cls_rel_loss\": 0.23833496096360424, \"loss\": 12.857303248511421, \"crf_ner_loss\": 8.788895103666517, \"crf_term_loss\": 3.8705077038870916, \"cls_rel_loss\": 0.19790057258473503, \"f1_ner\": 0.4642416769420469, \"f1_term\": 0.46090225563909765, \"f1_rel\": 0.5103503184713376, \"epoch\": 14, \"method\": \"With entity only\"}, {\"val_loss\": 13.87707072278879, \"val_crf_ner_loss\": 9.302793358364244, \"val_crf_term_loss\": 4.334407386988619, \"val_cls_rel_loss\": 0.23986962763932498, \"loss\": 11.917991108364529, \"crf_ner_loss\": 8.163964536454943, \"crf_term_loss\": 3.5885711246066623, \"cls_rel_loss\": 0.16545550607972676, \"f1_ner\": 0.4575202641849294, \"f1_term\": 0.46907817969661614, \"f1_rel\": 0.5349716446124764, \"epoch\": 15, \"method\": \"With entity only\"}, {\"val_loss\": 13.851704193727814, \"val_crf_ner_loss\": 9.281905269970858, \"val_crf_term_loss\": 4.333367421679253, \"val_cls_rel_loss\": 0.23643167219022765, \"loss\": 10.783254888322618, \"crf_ner_loss\": 7.348796447118123, \"crf_term_loss\": 3.2645195060306125, \"cls_rel_loss\": 0.16993894593583214, \"f1_ner\": 0.4778384942319369, \"f1_term\": 0.4589409056024559, \"f1_rel\": 0.5289183222958057, \"epoch\": 16, \"method\": \"With entity only\"}, {\"val_loss\": 13.83708848396357, \"val_crf_ner_loss\": 9.260113730047742, \"val_crf_term_loss\": 4.33385403486934, \"val_cls_rel_loss\": 0.24312052565769557, \"loss\": 13.709110021591187, \"crf_ner_loss\": 9.401966942681206, \"crf_term_loss\": 4.126997801992628, \"cls_rel_loss\": 0.18014505836698744, \"f1_ner\": 0.5148272699480281, \"f1_term\": 0.4684684684684685, \"f1_rel\": 0.526984126984127, \"epoch\": 17, \"method\": \"With entity only\"}, {\"val_loss\": 13.786796033817486, \"val_crf_ner_loss\": 9.23925695280089, \"val_crf_term_loss\": 4.309074413167299, \"val_cls_rel_loss\": 0.23846472701887145, \"loss\": 11.166400644514296, \"crf_ner_loss\": 7.633126152886285, \"crf_term_loss\": 3.3733496136135526, \"cls_rel_loss\": 0.15992481178707546, \"f1_ner\": 0.5514018691588785, \"f1_term\": 0.48030303030303034, \"f1_rel\": 0.5301629238221048, \"epoch\": 18, \"method\": \"With entity only\"}, {\"val_loss\": 13.805246923961779, \"val_crf_ner_loss\": 9.218766731067296, \"val_crf_term_loss\": 4.346495255936671, \"val_cls_rel_loss\": 0.23998485864513983, \"loss\": 11.829424328274197, \"crf_ner_loss\": 8.086160898208618, \"crf_term_loss\": 3.5691614283455744, \"cls_rel_loss\": 0.17410197026199764, \"f1_ner\": 0.548983924780103, \"f1_term\": 0.46465481603968584, \"f1_rel\": 0.5345402572653645, \"epoch\": 19, \"method\": \"With entity only\"}, {\"val_loss\": 13.806655104142907, \"val_crf_ner_loss\": 9.205200129181799, \"val_crf_term_loss\": 4.366030349348583, \"val_cls_rel_loss\": 0.23542456492020267, \"loss\": 12.593680832121107, \"crf_ner_loss\": 8.6165079275767, \"crf_term_loss\": 3.8064297437667847, \"cls_rel_loss\": 0.17074299852053323, \"f1_ner\": 0.5857970131057604, \"f1_term\": 0.45304347826086955, \"f1_rel\": 0.5360339142722562, \"epoch\": 20, \"method\": \"With entity only\"}, {\"val_loss\": 15.560061750620822, \"val_crf_ner_loss\": 10.219653251397348, \"val_crf_term_loss\": 4.834240241642416, \"val_cls_rel_loss\": 0.5061685951956867, \"loss\": 14.61849053700765, \"crf_ner_loss\": 10.024742497338188, \"crf_term_loss\": 4.184552060233222, \"cls_rel_loss\": 0.4091958983076943, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"All sentences\"}, {\"val_loss\": 15.198520618633632, \"val_crf_ner_loss\": 10.098720501809225, \"val_crf_term_loss\": 4.648429468600419, \"val_cls_rel_loss\": 0.45137060819751157, \"loss\": 14.341292593214247, \"crf_ner_loss\": 9.808199167251587, \"crf_term_loss\": 4.236886435084873, \"cls_rel_loss\": 0.2962071845928828, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 2, \"method\": \"All sentences\"}, {\"val_loss\": 14.928212701839252, \"val_crf_ner_loss\": 9.958176167341914, \"val_crf_term_loss\": 4.5732383902055505, \"val_cls_rel_loss\": 0.39679846711402394, \"loss\": 13.449421485265097, \"crf_ner_loss\": 9.17795599831475, \"crf_term_loss\": 3.9921832614474826, \"cls_rel_loss\": 0.2792823016643524, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.18351741716227699, \"epoch\": 3, \"method\": \"All sentences\"}, {\"val_loss\": 14.76241027873798, \"val_crf_ner_loss\": 9.874341665393244, \"val_crf_term_loss\": 4.516334484963521, \"val_cls_rel_loss\": 0.3717341934242388, \"loss\": 13.711050351460775, \"crf_ner_loss\": 9.378151390287611, \"crf_term_loss\": 4.080903225474888, \"cls_rel_loss\": 0.2519957605335448, \"f1_ner\": 0.0, \"f1_term\": 0.026195153896529145, \"f1_rel\": 0.13408876298394712, \"epoch\": 4, \"method\": \"All sentences\"}, {\"val_loss\": 14.577161350389467, \"val_crf_ner_loss\": 9.767085399070796, \"val_crf_term_loss\": 4.478528411719051, \"val_cls_rel_loss\": 0.3315476359677141, \"loss\": 13.33203911781311, \"crf_ner_loss\": 9.157547314961752, \"crf_term_loss\": 3.9552657339307995, \"cls_rel_loss\": 0.21922588100035986, \"f1_ner\": 0.0, \"f1_term\": 0.022696929238985312, \"f1_rel\": 0.3236009732360097, \"epoch\": 5, \"method\": \"All sentences\"}, {\"val_loss\": 14.43750627893601, \"val_crf_ner_loss\": 9.673909403111812, \"val_crf_term_loss\": 4.4633757284958, \"val_cls_rel_loss\": 0.3002214940795063, \"loss\": 14.043683263990614, \"crf_ner_loss\": 9.638826052347818, \"crf_term_loss\": 4.180516984727648, \"cls_rel_loss\": 0.22434028486410776, \"f1_ner\": 0.0, \"f1_term\": 0.02976995940460081, \"f1_rel\": 0.4169435215946844, \"epoch\": 6, \"method\": \"All sentences\"}, {\"val_loss\": 14.335392321983393, \"val_crf_ner_loss\": 9.601754004067748, \"val_crf_term_loss\": 4.454988237715115, \"val_cls_rel_loss\": 0.27864985352885113, \"loss\": 13.471335539550505, \"crf_ner_loss\": 9.256209160061468, \"crf_term_loss\": 4.0231675560608045, \"cls_rel_loss\": 0.19195865938072826, \"f1_ner\": 0.09342039176293321, \"f1_term\": 0.06107862248213125, \"f1_rel\": 0.4675767918088737, \"epoch\": 7, \"method\": \"All sentences\"}, {\"val_loss\": 14.248672032878346, \"val_crf_ner_loss\": 9.550206334051424, \"val_crf_term_loss\": 4.43272578629264, \"val_cls_rel_loss\": 0.2657397724416134, \"loss\": 10.864235109753078, \"crf_ner_loss\": 7.41835814052158, \"crf_term_loss\": 3.244785295592414, \"cls_rel_loss\": 0.20109177629152933, \"f1_ner\": 0.23877245508982034, \"f1_term\": 0.09577114427860696, \"f1_rel\": 0.49005424954792043, \"epoch\": 8, \"method\": \"All sentences\"}, {\"val_loss\": 14.169491339774027, \"val_crf_ner_loss\": 9.51067601503247, \"val_crf_term_loss\": 4.3926376635140745, \"val_cls_rel_loss\": 0.2661778715840221, \"loss\": 11.57474003897773, \"crf_ner_loss\": 7.915637572606404, \"crf_term_loss\": 3.47707642449273, \"cls_rel_loss\": 0.1820261784725719, \"f1_ner\": 0.27753743760399335, \"f1_term\": 0.1713106295149639, \"f1_rel\": 0.49172576832151305, \"epoch\": 9, \"method\": \"All sentences\"}, {\"val_loss\": 14.134371740104509, \"val_crf_ner_loss\": 9.47610334758341, \"val_crf_term_loss\": 4.3935526531108104, \"val_cls_rel_loss\": 0.2647157968395818, \"loss\": 12.72979211807251, \"crf_ner_loss\": 8.73153993818495, \"crf_term_loss\": 3.82661145263248, \"cls_rel_loss\": 0.17164075954092872, \"f1_ner\": 0.33240826395312983, \"f1_term\": 0.1515837104072398, \"f1_rel\": 0.484920634920635, \"epoch\": 10, \"method\": \"All sentences\"}, {\"val_loss\": 14.109744851606605, \"val_crf_ner_loss\": 9.441108084073031, \"val_crf_term_loss\": 4.409738464077024, \"val_cls_rel_loss\": 0.25889818859796454, \"loss\": 13.497358322143555, \"crf_ner_loss\": 9.285651869244045, \"crf_term_loss\": 4.041120608647664, \"cls_rel_loss\": 0.1705858620504538, \"f1_ner\": 0.35988926484158723, \"f1_term\": 0.19556566230812963, \"f1_rel\": 0.5080605764533463, \"epoch\": 11, \"method\": \"All sentences\"}, {\"val_loss\": 14.076229311253902, \"val_crf_ner_loss\": 9.408538179676029, \"val_crf_term_loss\": 4.402429541532141, \"val_cls_rel_loss\": 0.26526164660488605, \"loss\": 13.330087608761257, \"crf_ner_loss\": 9.17171483569675, \"crf_term_loss\": 3.995752837922838, \"cls_rel_loss\": 0.16262009574307335, \"f1_ner\": 0.40727957326639475, \"f1_term\": 0.2764673614920461, \"f1_rel\": 0.5076190476190476, \"epoch\": 12, \"method\": \"All sentences\"}, {\"val_loss\": 14.025057542063024, \"val_crf_ner_loss\": 9.383817965096801, \"val_crf_term_loss\": 4.3899574227576705, \"val_cls_rel_loss\": 0.2512820189016579, \"loss\": 10.935200108422173, \"crf_ner_loss\": 7.4790145556132, \"crf_term_loss\": 3.2752224736743503, \"cls_rel_loss\": 0.18096306009425056, \"f1_ner\": 0.43491767629698663, \"f1_term\": 0.33554987212276216, \"f1_rel\": 0.5099601593625498, \"epoch\": 13, \"method\": \"All sentences\"}, {\"val_loss\": 13.95655904199085, \"val_crf_ner_loss\": 9.359742932076001, \"val_crf_term_loss\": 4.346859073116832, \"val_cls_rel_loss\": 0.24995723257969765, \"loss\": 13.672428343031141, \"crf_ner_loss\": 9.416304641299778, \"crf_term_loss\": 4.106093181504144, \"cls_rel_loss\": 0.15003018039796087, \"f1_ner\": 0.4515727187791965, \"f1_term\": 0.3736749116607773, \"f1_rel\": 0.5114813147230977, \"epoch\": 14, \"method\": \"All sentences\"}, {\"val_loss\": 13.93107981577407, \"val_crf_ner_loss\": 9.336562083585418, \"val_crf_term_loss\": 4.343606795707759, \"val_cls_rel_loss\": 0.2509109221235679, \"loss\": 11.906590170330471, \"crf_ner_loss\": 8.182677851782906, \"crf_term_loss\": 3.573321567641364, \"cls_rel_loss\": 0.1505906908876366, \"f1_ner\": 0.47516198704103674, \"f1_term\": 0.40602836879432624, \"f1_rel\": 0.5275800711743772, \"epoch\": 15, \"method\": \"All sentences\"}, {\"val_loss\": 13.928269602086422, \"val_crf_ner_loss\": 9.317142190724393, \"val_crf_term_loss\": 4.366590955831708, \"val_cls_rel_loss\": 0.2445363311001854, \"loss\": 12.97074524561564, \"crf_ner_loss\": 8.915461487240261, \"crf_term_loss\": 3.8990012539757624, \"cls_rel_loss\": 0.15628258718384636, \"f1_ner\": 0.4880147510755992, \"f1_term\": 0.40456621004566207, \"f1_rel\": 0.5264562027567808, \"epoch\": 16, \"method\": \"All sentences\"}, {\"val_loss\": 13.899771871357938, \"val_crf_ner_loss\": 9.292585501705643, \"val_crf_term_loss\": 4.348387460639007, \"val_cls_rel_loss\": 0.2587989359876535, \"loss\": 13.465459373262194, \"crf_ner_loss\": 9.277238872316149, \"crf_term_loss\": 4.026697039604187, \"cls_rel_loss\": 0.16152334958314896, \"f1_ner\": 0.5033313143549364, \"f1_term\": 0.37316176470588236, \"f1_rel\": 0.5272895467160038, \"epoch\": 17, \"method\": \"All sentences\"}, {\"val_loss\": 13.900357114137524, \"val_crf_ner_loss\": 9.276182639337804, \"val_crf_term_loss\": 4.360796211409743, \"val_cls_rel_loss\": 0.26337817289533405, \"loss\": 11.405402104059855, \"crf_ner_loss\": 7.84924070040385, \"crf_term_loss\": 3.416629751523336, \"cls_rel_loss\": 0.13953178748488426, \"f1_ner\": 0.5406713032960387, \"f1_term\": 0.4014665444546288, \"f1_rel\": 0.5261277355962483, \"epoch\": 18, \"method\": \"All sentences\"}, {\"val_loss\": 13.896165350057784, \"val_crf_ner_loss\": 9.257388033135964, \"val_crf_term_loss\": 4.384082302559901, \"val_cls_rel_loss\": 0.2546948533858696, \"loss\": 11.414863692389595, \"crf_ner_loss\": 7.842244174745348, \"crf_term_loss\": 3.435810407002767, \"cls_rel_loss\": 0.13680922778116333, \"f1_ner\": 0.5773386847792529, \"f1_term\": 0.3829145728643216, \"f1_rel\": 0.5267175572519084, \"epoch\": 19, \"method\": \"All sentences\"}, {\"val_loss\": 13.877817710820775, \"val_crf_ner_loss\": 9.246109692719731, \"val_crf_term_loss\": 4.375795293028337, \"val_cls_rel_loss\": 0.255912614564826, \"loss\": 13.860608206854927, \"crf_ner_loss\": 9.544105344348484, \"crf_term_loss\": 4.17823080221812, \"cls_rel_loss\": 0.13827224034402105, \"f1_ner\": 0.5676408556794216, \"f1_term\": 0.3892750744786495, \"f1_rel\": 0.5167381974248927, \"epoch\": 20, \"method\": \"All sentences\"}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.HConcatChart(...)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "te_results_df = results_df[results_df[\"method\"].isin([\"KPM\", \"KPM*\"])].copy()\n",
    "te_results_df[\"method\"] = te_results_df[\"method\"].map({\"KPM\": \"All sentences\", \"KPM*\": \"With entity only\"})\n",
    "\n",
    "ner_loss = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"val_crf_ner_loss:Q\", title=\"NER Loss\", scale=alt.Scale(domain=[9, 10.4])),\n",
    "    color=alt.Color(\"method:N\", title=\"Training Set\")\n",
    ")\n",
    "\n",
    "term_loss = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"val_crf_term_loss:Q\", title=\"EE Loss\", scale=alt.Scale(domain=[4, 5])),\n",
    "    color=alt.Color(\"method:N\", title=\"Training Set\")\n",
    ")\n",
    "\n",
    "rel_loss = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"val_cls_rel_loss:Q\", title=\"RE Loss\", scale=alt.Scale(domain=[0, 0.6])),\n",
    "    color=alt.Color(\"method:N\", title=\"Training Set\")\n",
    ")\n",
    "\n",
    "all_charts = ner_loss | term_loss | rel_loss\n",
    "\n",
    "all_charts.configure_axis(\n",
    "    titleFont=\"Times New Roman\",\n",
    "    titleFontSize=18,\n",
    "    labelFont=\"Times New Roman\",\n",
    "    labelFontSize=18\n",
    ").configure_legend(\n",
    "    titleFont=\"Times New Roman\",\n",
    "    titleFontSize=20,\n",
    "    labelFont=\"Times New Roman\",\n",
    "    labelFontSize=18\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-7b213360049b48ed83f87711b50dabb5\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-7b213360049b48ed83f87711b50dabb5\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-7b213360049b48ed83f87711b50dabb5\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}, \"axis\": {\"labelFont\": \"Times New Roman\", \"labelFontSize\": 18, \"titleFont\": \"Times New Roman\", \"titleFontSize\": 18}, \"legend\": {\"labelFont\": \"Times New Roman\", \"labelFontSize\": 18, \"titleFont\": \"Times New Roman\", \"titleFontSize\": 20}}, \"hconcat\": [{\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"Training Set\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"f1_ner\", \"scale\": {\"domain\": [0, 0.6]}, \"title\": \"NER F-score\"}}}, {\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"Training Set\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"f1_term\", \"scale\": {\"domain\": [0, 0.6]}, \"title\": \"EE F-score\"}}}, {\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"Training Set\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"f1_rel\", \"scale\": {\"domain\": [0, 0.6]}, \"title\": \"RE F-score\"}}}], \"data\": {\"name\": \"data-1e9ecb5697c5ac9bc28637dbf161d151\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-1e9ecb5697c5ac9bc28637dbf161d151\": [{\"val_loss\": 15.301897678932134, \"val_crf_ner_loss\": 10.182450364105893, \"val_crf_term_loss\": 4.664121624327054, \"val_cls_rel_loss\": 0.4553257749028449, \"loss\": 13.774118158552382, \"crf_ner_loss\": 9.44224182764689, \"crf_term_loss\": 3.892859763569302, \"cls_rel_loss\": 0.43901652925544316, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"With entity only\"}, {\"val_loss\": 15.088710158410734, \"val_crf_ner_loss\": 10.042704025324245, \"val_crf_term_loss\": 4.624338203973144, \"val_cls_rel_loss\": 0.4216679147560231, \"loss\": 14.391434563530815, \"crf_ner_loss\": 9.721958584255642, \"crf_term_loss\": 4.309103541904026, \"cls_rel_loss\": 0.3603723694880803, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 2, \"method\": \"With entity only\"}, {\"val_loss\": 14.868212205650163, \"val_crf_ner_loss\": 9.94457231994963, \"val_crf_term_loss\": 4.530394581112549, \"val_cls_rel_loss\": 0.3932453250362925, \"loss\": 12.414401796129015, \"crf_ner_loss\": 8.338009569380018, \"crf_term_loss\": 3.7816480265723333, \"cls_rel_loss\": 0.29474418196413255, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.021929824561403508, \"epoch\": 3, \"method\": \"With entity only\"}, {\"val_loss\": 14.644684871617894, \"val_crf_ner_loss\": 9.838034796888811, \"val_crf_term_loss\": 4.464446843975652, \"val_cls_rel_loss\": 0.3422032877476546, \"loss\": 12.452283435397678, \"crf_ner_loss\": 8.426005283991495, \"crf_term_loss\": 3.7280285490883722, \"cls_rel_loss\": 0.29824961970249814, \"f1_ner\": 0.0, \"f1_term\": 0.012371134020618556, \"f1_rel\": 0.33200531208499334, \"epoch\": 4, \"method\": \"With entity only\"}, {\"val_loss\": 14.48356381646038, \"val_crf_ner_loss\": 9.733082833951407, \"val_crf_term_loss\": 4.437290826853174, \"val_cls_rel_loss\": 0.3131902344035406, \"loss\": 12.599537902408176, \"crf_ner_loss\": 8.55139226383633, \"crf_term_loss\": 3.790961265563965, \"cls_rel_loss\": 0.2571844566199515, \"f1_ner\": 0.0, \"f1_term\": 0.031862745098039214, \"f1_rel\": 0.33602347762289064, \"epoch\": 5, \"method\": \"With entity only\"}, {\"val_loss\": 14.285567158330096, \"val_crf_ner_loss\": 9.62895664333427, \"val_crf_term_loss\": 4.376269838235674, \"val_cls_rel_loss\": 0.28034074550127464, \"loss\": 11.80238119761149, \"crf_ner_loss\": 8.021390702989367, \"crf_term_loss\": 3.538855208290948, \"cls_rel_loss\": 0.24213523831632403, \"f1_ner\": 0.03489640130861505, \"f1_term\": 0.1641949152542373, \"f1_rel\": 0.4579710144927536, \"epoch\": 6, \"method\": \"With entity only\"}, {\"val_loss\": 14.190792644110909, \"val_crf_ner_loss\": 9.552197621686615, \"val_crf_term_loss\": 4.374607987647509, \"val_cls_rel_loss\": 0.2639873599919089, \"loss\": 15.422736909654406, \"crf_ner_loss\": 10.54382954703437, \"crf_term_loss\": 4.651935312483046, \"cls_rel_loss\": 0.22697221653329003, \"f1_ner\": 0.2111237230419977, \"f1_term\": 0.24483917426788285, \"f1_rel\": 0.4822411205602801, \"epoch\": 7, \"method\": \"With entity only\"}, {\"val_loss\": 14.100167079563558, \"val_crf_ner_loss\": 9.497426730872942, \"val_crf_term_loss\": 4.346293905355634, \"val_cls_rel_loss\": 0.2564464663937144, \"loss\": 13.520169470045301, \"crf_ner_loss\": 9.212770409054226, \"crf_term_loss\": 4.079206003083123, \"cls_rel_loss\": 0.2281930512852139, \"f1_ner\": 0.2951269732326699, \"f1_term\": 0.34651447207501024, \"f1_rel\": 0.48416112631990615, \"epoch\": 8, \"method\": \"With entity only\"}, {\"val_loss\": 14.051934238767972, \"val_crf_ner_loss\": 9.458889895111975, \"val_crf_term_loss\": 4.336595517875504, \"val_cls_rel_loss\": 0.25644875138345424, \"loss\": 12.486466672685411, \"crf_ner_loss\": 8.513386329015097, \"crf_term_loss\": 3.770048565334744, \"cls_rel_loss\": 0.20303168065018123, \"f1_ner\": 0.34801624492346145, \"f1_term\": 0.3886328725038403, \"f1_rel\": 0.5049739459971578, \"epoch\": 9, \"method\": \"With entity only\"}, {\"val_loss\": 14.029026644073264, \"val_crf_ner_loss\": 9.425868076129552, \"val_crf_term_loss\": 4.339041190425845, \"val_cls_rel_loss\": 0.2641175667734912, \"loss\": 11.872677952105645, \"crf_ner_loss\": 8.10128795533991, \"crf_term_loss\": 3.5802190727681955, \"cls_rel_loss\": 0.19117092780455594, \"f1_ner\": 0.4203262233375157, \"f1_term\": 0.4071246819338422, \"f1_rel\": 0.4720161834120027, \"epoch\": 10, \"method\": \"With entity only\"}, {\"val_loss\": 13.972422105552507, \"val_crf_ner_loss\": 9.398135232229302, \"val_crf_term_loss\": 4.321064592277916, \"val_cls_rel_loss\": 0.2532223985142951, \"loss\": 12.673591242896187, \"crf_ner_loss\": 8.674006409115261, \"crf_term_loss\": 3.8085707426071167, \"cls_rel_loss\": 0.19101394216219583, \"f1_ner\": 0.4384251968503937, \"f1_term\": 0.454308093994778, \"f1_rel\": 0.5206316861946001, \"epoch\": 11, \"method\": \"With entity only\"}, {\"val_loss\": 13.9664681636504, \"val_crf_ner_loss\": 9.375703832528886, \"val_crf_term_loss\": 4.348035911573981, \"val_cls_rel_loss\": 0.2427283042538775, \"loss\": 11.879486507839626, \"crf_ner_loss\": 8.127701388465034, \"crf_term_loss\": 3.56903256310357, \"cls_rel_loss\": 0.18275263408819833, \"f1_ner\": 0.41350210970464135, \"f1_term\": 0.44039603960396045, \"f1_rel\": 0.5142371440713982, \"epoch\": 12, \"method\": \"With entity only\"}, {\"val_loss\": 13.93406541504129, \"val_crf_ner_loss\": 9.349364701848831, \"val_crf_term_loss\": 4.344773184644045, \"val_cls_rel_loss\": 0.23992731427624278, \"loss\": 12.419934537675646, \"crf_ner_loss\": 8.50323814815945, \"crf_term_loss\": 3.7242375877168445, \"cls_rel_loss\": 0.19245871818727917, \"f1_ner\": 0.44039530574428665, \"f1_term\": 0.4196141479099679, \"f1_rel\": 0.5310245310245311, \"epoch\": 13, \"method\": \"With entity only\"}, {\"val_loss\": 13.889922232523451, \"val_crf_ner_loss\": 9.325161039394184, \"val_crf_term_loss\": 4.326426059660251, \"val_cls_rel_loss\": 0.23833496096360424, \"loss\": 12.857303248511421, \"crf_ner_loss\": 8.788895103666517, \"crf_term_loss\": 3.8705077038870916, \"cls_rel_loss\": 0.19790057258473503, \"f1_ner\": 0.4642416769420469, \"f1_term\": 0.46090225563909765, \"f1_rel\": 0.5103503184713376, \"epoch\": 14, \"method\": \"With entity only\"}, {\"val_loss\": 13.87707072278879, \"val_crf_ner_loss\": 9.302793358364244, \"val_crf_term_loss\": 4.334407386988619, \"val_cls_rel_loss\": 0.23986962763932498, \"loss\": 11.917991108364529, \"crf_ner_loss\": 8.163964536454943, \"crf_term_loss\": 3.5885711246066623, \"cls_rel_loss\": 0.16545550607972676, \"f1_ner\": 0.4575202641849294, \"f1_term\": 0.46907817969661614, \"f1_rel\": 0.5349716446124764, \"epoch\": 15, \"method\": \"With entity only\"}, {\"val_loss\": 13.851704193727814, \"val_crf_ner_loss\": 9.281905269970858, \"val_crf_term_loss\": 4.333367421679253, \"val_cls_rel_loss\": 0.23643167219022765, \"loss\": 10.783254888322618, \"crf_ner_loss\": 7.348796447118123, \"crf_term_loss\": 3.2645195060306125, \"cls_rel_loss\": 0.16993894593583214, \"f1_ner\": 0.4778384942319369, \"f1_term\": 0.4589409056024559, \"f1_rel\": 0.5289183222958057, \"epoch\": 16, \"method\": \"With entity only\"}, {\"val_loss\": 13.83708848396357, \"val_crf_ner_loss\": 9.260113730047742, \"val_crf_term_loss\": 4.33385403486934, \"val_cls_rel_loss\": 0.24312052565769557, \"loss\": 13.709110021591187, \"crf_ner_loss\": 9.401966942681206, \"crf_term_loss\": 4.126997801992628, \"cls_rel_loss\": 0.18014505836698744, \"f1_ner\": 0.5148272699480281, \"f1_term\": 0.4684684684684685, \"f1_rel\": 0.526984126984127, \"epoch\": 17, \"method\": \"With entity only\"}, {\"val_loss\": 13.786796033817486, \"val_crf_ner_loss\": 9.23925695280089, \"val_crf_term_loss\": 4.309074413167299, \"val_cls_rel_loss\": 0.23846472701887145, \"loss\": 11.166400644514296, \"crf_ner_loss\": 7.633126152886285, \"crf_term_loss\": 3.3733496136135526, \"cls_rel_loss\": 0.15992481178707546, \"f1_ner\": 0.5514018691588785, \"f1_term\": 0.48030303030303034, \"f1_rel\": 0.5301629238221048, \"epoch\": 18, \"method\": \"With entity only\"}, {\"val_loss\": 13.805246923961779, \"val_crf_ner_loss\": 9.218766731067296, \"val_crf_term_loss\": 4.346495255936671, \"val_cls_rel_loss\": 0.23998485864513983, \"loss\": 11.829424328274197, \"crf_ner_loss\": 8.086160898208618, \"crf_term_loss\": 3.5691614283455744, \"cls_rel_loss\": 0.17410197026199764, \"f1_ner\": 0.548983924780103, \"f1_term\": 0.46465481603968584, \"f1_rel\": 0.5345402572653645, \"epoch\": 19, \"method\": \"With entity only\"}, {\"val_loss\": 13.806655104142907, \"val_crf_ner_loss\": 9.205200129181799, \"val_crf_term_loss\": 4.366030349348583, \"val_cls_rel_loss\": 0.23542456492020267, \"loss\": 12.593680832121107, \"crf_ner_loss\": 8.6165079275767, \"crf_term_loss\": 3.8064297437667847, \"cls_rel_loss\": 0.17074299852053323, \"f1_ner\": 0.5857970131057604, \"f1_term\": 0.45304347826086955, \"f1_rel\": 0.5360339142722562, \"epoch\": 20, \"method\": \"With entity only\"}, {\"val_loss\": 15.560061750620822, \"val_crf_ner_loss\": 10.219653251397348, \"val_crf_term_loss\": 4.834240241642416, \"val_cls_rel_loss\": 0.5061685951956867, \"loss\": 14.61849053700765, \"crf_ner_loss\": 10.024742497338188, \"crf_term_loss\": 4.184552060233222, \"cls_rel_loss\": 0.4091958983076943, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"All sentences\"}, {\"val_loss\": 15.198520618633632, \"val_crf_ner_loss\": 10.098720501809225, \"val_crf_term_loss\": 4.648429468600419, \"val_cls_rel_loss\": 0.45137060819751157, \"loss\": 14.341292593214247, \"crf_ner_loss\": 9.808199167251587, \"crf_term_loss\": 4.236886435084873, \"cls_rel_loss\": 0.2962071845928828, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 2, \"method\": \"All sentences\"}, {\"val_loss\": 14.928212701839252, \"val_crf_ner_loss\": 9.958176167341914, \"val_crf_term_loss\": 4.5732383902055505, \"val_cls_rel_loss\": 0.39679846711402394, \"loss\": 13.449421485265097, \"crf_ner_loss\": 9.17795599831475, \"crf_term_loss\": 3.9921832614474826, \"cls_rel_loss\": 0.2792823016643524, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.18351741716227699, \"epoch\": 3, \"method\": \"All sentences\"}, {\"val_loss\": 14.76241027873798, \"val_crf_ner_loss\": 9.874341665393244, \"val_crf_term_loss\": 4.516334484963521, \"val_cls_rel_loss\": 0.3717341934242388, \"loss\": 13.711050351460775, \"crf_ner_loss\": 9.378151390287611, \"crf_term_loss\": 4.080903225474888, \"cls_rel_loss\": 0.2519957605335448, \"f1_ner\": 0.0, \"f1_term\": 0.026195153896529145, \"f1_rel\": 0.13408876298394712, \"epoch\": 4, \"method\": \"All sentences\"}, {\"val_loss\": 14.577161350389467, \"val_crf_ner_loss\": 9.767085399070796, \"val_crf_term_loss\": 4.478528411719051, \"val_cls_rel_loss\": 0.3315476359677141, \"loss\": 13.33203911781311, \"crf_ner_loss\": 9.157547314961752, \"crf_term_loss\": 3.9552657339307995, \"cls_rel_loss\": 0.21922588100035986, \"f1_ner\": 0.0, \"f1_term\": 0.022696929238985312, \"f1_rel\": 0.3236009732360097, \"epoch\": 5, \"method\": \"All sentences\"}, {\"val_loss\": 14.43750627893601, \"val_crf_ner_loss\": 9.673909403111812, \"val_crf_term_loss\": 4.4633757284958, \"val_cls_rel_loss\": 0.3002214940795063, \"loss\": 14.043683263990614, \"crf_ner_loss\": 9.638826052347818, \"crf_term_loss\": 4.180516984727648, \"cls_rel_loss\": 0.22434028486410776, \"f1_ner\": 0.0, \"f1_term\": 0.02976995940460081, \"f1_rel\": 0.4169435215946844, \"epoch\": 6, \"method\": \"All sentences\"}, {\"val_loss\": 14.335392321983393, \"val_crf_ner_loss\": 9.601754004067748, \"val_crf_term_loss\": 4.454988237715115, \"val_cls_rel_loss\": 0.27864985352885113, \"loss\": 13.471335539550505, \"crf_ner_loss\": 9.256209160061468, \"crf_term_loss\": 4.0231675560608045, \"cls_rel_loss\": 0.19195865938072826, \"f1_ner\": 0.09342039176293321, \"f1_term\": 0.06107862248213125, \"f1_rel\": 0.4675767918088737, \"epoch\": 7, \"method\": \"All sentences\"}, {\"val_loss\": 14.248672032878346, \"val_crf_ner_loss\": 9.550206334051424, \"val_crf_term_loss\": 4.43272578629264, \"val_cls_rel_loss\": 0.2657397724416134, \"loss\": 10.864235109753078, \"crf_ner_loss\": 7.41835814052158, \"crf_term_loss\": 3.244785295592414, \"cls_rel_loss\": 0.20109177629152933, \"f1_ner\": 0.23877245508982034, \"f1_term\": 0.09577114427860696, \"f1_rel\": 0.49005424954792043, \"epoch\": 8, \"method\": \"All sentences\"}, {\"val_loss\": 14.169491339774027, \"val_crf_ner_loss\": 9.51067601503247, \"val_crf_term_loss\": 4.3926376635140745, \"val_cls_rel_loss\": 0.2661778715840221, \"loss\": 11.57474003897773, \"crf_ner_loss\": 7.915637572606404, \"crf_term_loss\": 3.47707642449273, \"cls_rel_loss\": 0.1820261784725719, \"f1_ner\": 0.27753743760399335, \"f1_term\": 0.1713106295149639, \"f1_rel\": 0.49172576832151305, \"epoch\": 9, \"method\": \"All sentences\"}, {\"val_loss\": 14.134371740104509, \"val_crf_ner_loss\": 9.47610334758341, \"val_crf_term_loss\": 4.3935526531108104, \"val_cls_rel_loss\": 0.2647157968395818, \"loss\": 12.72979211807251, \"crf_ner_loss\": 8.73153993818495, \"crf_term_loss\": 3.82661145263248, \"cls_rel_loss\": 0.17164075954092872, \"f1_ner\": 0.33240826395312983, \"f1_term\": 0.1515837104072398, \"f1_rel\": 0.484920634920635, \"epoch\": 10, \"method\": \"All sentences\"}, {\"val_loss\": 14.109744851606605, \"val_crf_ner_loss\": 9.441108084073031, \"val_crf_term_loss\": 4.409738464077024, \"val_cls_rel_loss\": 0.25889818859796454, \"loss\": 13.497358322143555, \"crf_ner_loss\": 9.285651869244045, \"crf_term_loss\": 4.041120608647664, \"cls_rel_loss\": 0.1705858620504538, \"f1_ner\": 0.35988926484158723, \"f1_term\": 0.19556566230812963, \"f1_rel\": 0.5080605764533463, \"epoch\": 11, \"method\": \"All sentences\"}, {\"val_loss\": 14.076229311253902, \"val_crf_ner_loss\": 9.408538179676029, \"val_crf_term_loss\": 4.402429541532141, \"val_cls_rel_loss\": 0.26526164660488605, \"loss\": 13.330087608761257, \"crf_ner_loss\": 9.17171483569675, \"crf_term_loss\": 3.995752837922838, \"cls_rel_loss\": 0.16262009574307335, \"f1_ner\": 0.40727957326639475, \"f1_term\": 0.2764673614920461, \"f1_rel\": 0.5076190476190476, \"epoch\": 12, \"method\": \"All sentences\"}, {\"val_loss\": 14.025057542063024, \"val_crf_ner_loss\": 9.383817965096801, \"val_crf_term_loss\": 4.3899574227576705, \"val_cls_rel_loss\": 0.2512820189016579, \"loss\": 10.935200108422173, \"crf_ner_loss\": 7.4790145556132, \"crf_term_loss\": 3.2752224736743503, \"cls_rel_loss\": 0.18096306009425056, \"f1_ner\": 0.43491767629698663, \"f1_term\": 0.33554987212276216, \"f1_rel\": 0.5099601593625498, \"epoch\": 13, \"method\": \"All sentences\"}, {\"val_loss\": 13.95655904199085, \"val_crf_ner_loss\": 9.359742932076001, \"val_crf_term_loss\": 4.346859073116832, \"val_cls_rel_loss\": 0.24995723257969765, \"loss\": 13.672428343031141, \"crf_ner_loss\": 9.416304641299778, \"crf_term_loss\": 4.106093181504144, \"cls_rel_loss\": 0.15003018039796087, \"f1_ner\": 0.4515727187791965, \"f1_term\": 0.3736749116607773, \"f1_rel\": 0.5114813147230977, \"epoch\": 14, \"method\": \"All sentences\"}, {\"val_loss\": 13.93107981577407, \"val_crf_ner_loss\": 9.336562083585418, \"val_crf_term_loss\": 4.343606795707759, \"val_cls_rel_loss\": 0.2509109221235679, \"loss\": 11.906590170330471, \"crf_ner_loss\": 8.182677851782906, \"crf_term_loss\": 3.573321567641364, \"cls_rel_loss\": 0.1505906908876366, \"f1_ner\": 0.47516198704103674, \"f1_term\": 0.40602836879432624, \"f1_rel\": 0.5275800711743772, \"epoch\": 15, \"method\": \"All sentences\"}, {\"val_loss\": 13.928269602086422, \"val_crf_ner_loss\": 9.317142190724393, \"val_crf_term_loss\": 4.366590955831708, \"val_cls_rel_loss\": 0.2445363311001854, \"loss\": 12.97074524561564, \"crf_ner_loss\": 8.915461487240261, \"crf_term_loss\": 3.8990012539757624, \"cls_rel_loss\": 0.15628258718384636, \"f1_ner\": 0.4880147510755992, \"f1_term\": 0.40456621004566207, \"f1_rel\": 0.5264562027567808, \"epoch\": 16, \"method\": \"All sentences\"}, {\"val_loss\": 13.899771871357938, \"val_crf_ner_loss\": 9.292585501705643, \"val_crf_term_loss\": 4.348387460639007, \"val_cls_rel_loss\": 0.2587989359876535, \"loss\": 13.465459373262194, \"crf_ner_loss\": 9.277238872316149, \"crf_term_loss\": 4.026697039604187, \"cls_rel_loss\": 0.16152334958314896, \"f1_ner\": 0.5033313143549364, \"f1_term\": 0.37316176470588236, \"f1_rel\": 0.5272895467160038, \"epoch\": 17, \"method\": \"All sentences\"}, {\"val_loss\": 13.900357114137524, \"val_crf_ner_loss\": 9.276182639337804, \"val_crf_term_loss\": 4.360796211409743, \"val_cls_rel_loss\": 0.26337817289533405, \"loss\": 11.405402104059855, \"crf_ner_loss\": 7.84924070040385, \"crf_term_loss\": 3.416629751523336, \"cls_rel_loss\": 0.13953178748488426, \"f1_ner\": 0.5406713032960387, \"f1_term\": 0.4014665444546288, \"f1_rel\": 0.5261277355962483, \"epoch\": 18, \"method\": \"All sentences\"}, {\"val_loss\": 13.896165350057784, \"val_crf_ner_loss\": 9.257388033135964, \"val_crf_term_loss\": 4.384082302559901, \"val_cls_rel_loss\": 0.2546948533858696, \"loss\": 11.414863692389595, \"crf_ner_loss\": 7.842244174745348, \"crf_term_loss\": 3.435810407002767, \"cls_rel_loss\": 0.13680922778116333, \"f1_ner\": 0.5773386847792529, \"f1_term\": 0.3829145728643216, \"f1_rel\": 0.5267175572519084, \"epoch\": 19, \"method\": \"All sentences\"}, {\"val_loss\": 13.877817710820775, \"val_crf_ner_loss\": 9.246109692719731, \"val_crf_term_loss\": 4.375795293028337, \"val_cls_rel_loss\": 0.255912614564826, \"loss\": 13.860608206854927, \"crf_ner_loss\": 9.544105344348484, \"crf_term_loss\": 4.17823080221812, \"cls_rel_loss\": 0.13827224034402105, \"f1_ner\": 0.5676408556794216, \"f1_term\": 0.3892750744786495, \"f1_rel\": 0.5167381974248927, \"epoch\": 20, \"method\": \"All sentences\"}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.HConcatChart(...)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "te_results_df = results_df[results_df[\"method\"].isin([\"KPM\", \"KPM*\"])].copy()\n",
    "te_results_df[\"method\"] = te_results_df[\"method\"].map({\"KPM\": \"All sentences\", \"KPM*\": \"With entity only\"})\n",
    "\n",
    "ner_f1 = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"f1_ner:Q\", title=\"NER F-score\", scale=alt.Scale(domain=[0, 0.6])),\n",
    "    color=alt.Color(\"method:N\", title=\"Training Set\")\n",
    ")\n",
    "\n",
    "term_f1 = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"f1_term:Q\", title=\"EE F-score\", scale=alt.Scale(domain=[0, 0.6])),\n",
    "    color=alt.Color(\"method:N\", title=\"Training Set\")\n",
    ")\n",
    "\n",
    "rel_f1 = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"f1_rel:Q\", title=\"RE F-score\", scale=alt.Scale(domain=[0, 0.6])),\n",
    "    color=alt.Color(\"method:N\", title=\"Training Set\")\n",
    ")\n",
    "\n",
    "all_charts = ner_f1 | term_f1 | rel_f1\n",
    "\n",
    "all_charts.configure_axis(\n",
    "    titleFont=\"Times New Roman\",\n",
    "    titleFontSize=18,\n",
    "    labelFont=\"Times New Roman\",\n",
    "    labelFontSize=18\n",
    ").configure_legend(\n",
    "    titleFont=\"Times New Roman\",\n",
    "    titleFontSize=20,\n",
    "    labelFont=\"Times New Roman\",\n",
    "    labelFontSize=18\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-ebd044def4074b049b8025a1d00f60de\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-ebd044def4074b049b8025a1d00f60de\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-ebd044def4074b049b8025a1d00f60de\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}, \"axis\": {\"labelFont\": \"Times New Roman\", \"labelFontSize\": 18, \"titleFont\": \"Times New Roman\", \"titleFontSize\": 18}, \"legend\": {\"labelFont\": \"Times New Roman\", \"labelFontSize\": 18, \"titleFont\": \"Times New Roman\", \"titleFontSize\": 20}}, \"hconcat\": [{\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"TE Method\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"val_crf_ner_loss\", \"scale\": {\"domain\": [9, 10.4]}, \"title\": \"NER Loss\"}}}, {\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"TE Method\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"val_crf_term_loss\", \"scale\": {\"domain\": [4, 5]}, \"title\": \"TE Loss\"}}}, {\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"TE Method\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"val_cls_rel_loss\", \"scale\": {\"domain\": [0, 0.6]}, \"title\": \"RE Loss\"}}}], \"data\": {\"name\": \"data-d179cc84f596b886aeafa9ac8a80335d\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-d179cc84f596b886aeafa9ac8a80335d\": [{\"val_loss\": 15.301897678932134, \"val_crf_ner_loss\": 10.182450364105893, \"val_crf_term_loss\": 4.664121624327054, \"val_cls_rel_loss\": 0.4553257749028449, \"loss\": 13.774118158552382, \"crf_ner_loss\": 9.44224182764689, \"crf_term_loss\": 3.892859763569302, \"cls_rel_loss\": 0.43901652925544316, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"KPM*\"}, {\"val_loss\": 15.088710158410734, \"val_crf_ner_loss\": 10.042704025324245, \"val_crf_term_loss\": 4.624338203973144, \"val_cls_rel_loss\": 0.4216679147560231, \"loss\": 14.391434563530815, \"crf_ner_loss\": 9.721958584255642, \"crf_term_loss\": 4.309103541904026, \"cls_rel_loss\": 0.3603723694880803, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 2, \"method\": \"KPM*\"}, {\"val_loss\": 14.868212205650163, \"val_crf_ner_loss\": 9.94457231994963, \"val_crf_term_loss\": 4.530394581112549, \"val_cls_rel_loss\": 0.3932453250362925, \"loss\": 12.414401796129015, \"crf_ner_loss\": 8.338009569380018, \"crf_term_loss\": 3.7816480265723333, \"cls_rel_loss\": 0.29474418196413255, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.021929824561403508, \"epoch\": 3, \"method\": \"KPM*\"}, {\"val_loss\": 14.644684871617894, \"val_crf_ner_loss\": 9.838034796888811, \"val_crf_term_loss\": 4.464446843975652, \"val_cls_rel_loss\": 0.3422032877476546, \"loss\": 12.452283435397678, \"crf_ner_loss\": 8.426005283991495, \"crf_term_loss\": 3.7280285490883722, \"cls_rel_loss\": 0.29824961970249814, \"f1_ner\": 0.0, \"f1_term\": 0.012371134020618556, \"f1_rel\": 0.33200531208499334, \"epoch\": 4, \"method\": \"KPM*\"}, {\"val_loss\": 14.48356381646038, \"val_crf_ner_loss\": 9.733082833951407, \"val_crf_term_loss\": 4.437290826853174, \"val_cls_rel_loss\": 0.3131902344035406, \"loss\": 12.599537902408176, \"crf_ner_loss\": 8.55139226383633, \"crf_term_loss\": 3.790961265563965, \"cls_rel_loss\": 0.2571844566199515, \"f1_ner\": 0.0, \"f1_term\": 0.031862745098039214, \"f1_rel\": 0.33602347762289064, \"epoch\": 5, \"method\": \"KPM*\"}, {\"val_loss\": 14.285567158330096, \"val_crf_ner_loss\": 9.62895664333427, \"val_crf_term_loss\": 4.376269838235674, \"val_cls_rel_loss\": 0.28034074550127464, \"loss\": 11.80238119761149, \"crf_ner_loss\": 8.021390702989367, \"crf_term_loss\": 3.538855208290948, \"cls_rel_loss\": 0.24213523831632403, \"f1_ner\": 0.03489640130861505, \"f1_term\": 0.1641949152542373, \"f1_rel\": 0.4579710144927536, \"epoch\": 6, \"method\": \"KPM*\"}, {\"val_loss\": 14.190792644110909, \"val_crf_ner_loss\": 9.552197621686615, \"val_crf_term_loss\": 4.374607987647509, \"val_cls_rel_loss\": 0.2639873599919089, \"loss\": 15.422736909654406, \"crf_ner_loss\": 10.54382954703437, \"crf_term_loss\": 4.651935312483046, \"cls_rel_loss\": 0.22697221653329003, \"f1_ner\": 0.2111237230419977, \"f1_term\": 0.24483917426788285, \"f1_rel\": 0.4822411205602801, \"epoch\": 7, \"method\": \"KPM*\"}, {\"val_loss\": 14.100167079563558, \"val_crf_ner_loss\": 9.497426730872942, \"val_crf_term_loss\": 4.346293905355634, \"val_cls_rel_loss\": 0.2564464663937144, \"loss\": 13.520169470045301, \"crf_ner_loss\": 9.212770409054226, \"crf_term_loss\": 4.079206003083123, \"cls_rel_loss\": 0.2281930512852139, \"f1_ner\": 0.2951269732326699, \"f1_term\": 0.34651447207501024, \"f1_rel\": 0.48416112631990615, \"epoch\": 8, \"method\": \"KPM*\"}, {\"val_loss\": 14.051934238767972, \"val_crf_ner_loss\": 9.458889895111975, \"val_crf_term_loss\": 4.336595517875504, \"val_cls_rel_loss\": 0.25644875138345424, \"loss\": 12.486466672685411, \"crf_ner_loss\": 8.513386329015097, \"crf_term_loss\": 3.770048565334744, \"cls_rel_loss\": 0.20303168065018123, \"f1_ner\": 0.34801624492346145, \"f1_term\": 0.3886328725038403, \"f1_rel\": 0.5049739459971578, \"epoch\": 9, \"method\": \"KPM*\"}, {\"val_loss\": 14.029026644073264, \"val_crf_ner_loss\": 9.425868076129552, \"val_crf_term_loss\": 4.339041190425845, \"val_cls_rel_loss\": 0.2641175667734912, \"loss\": 11.872677952105645, \"crf_ner_loss\": 8.10128795533991, \"crf_term_loss\": 3.5802190727681955, \"cls_rel_loss\": 0.19117092780455594, \"f1_ner\": 0.4203262233375157, \"f1_term\": 0.4071246819338422, \"f1_rel\": 0.4720161834120027, \"epoch\": 10, \"method\": \"KPM*\"}, {\"val_loss\": 13.972422105552507, \"val_crf_ner_loss\": 9.398135232229302, \"val_crf_term_loss\": 4.321064592277916, \"val_cls_rel_loss\": 0.2532223985142951, \"loss\": 12.673591242896187, \"crf_ner_loss\": 8.674006409115261, \"crf_term_loss\": 3.8085707426071167, \"cls_rel_loss\": 0.19101394216219583, \"f1_ner\": 0.4384251968503937, \"f1_term\": 0.454308093994778, \"f1_rel\": 0.5206316861946001, \"epoch\": 11, \"method\": \"KPM*\"}, {\"val_loss\": 13.9664681636504, \"val_crf_ner_loss\": 9.375703832528886, \"val_crf_term_loss\": 4.348035911573981, \"val_cls_rel_loss\": 0.2427283042538775, \"loss\": 11.879486507839626, \"crf_ner_loss\": 8.127701388465034, \"crf_term_loss\": 3.56903256310357, \"cls_rel_loss\": 0.18275263408819833, \"f1_ner\": 0.41350210970464135, \"f1_term\": 0.44039603960396045, \"f1_rel\": 0.5142371440713982, \"epoch\": 12, \"method\": \"KPM*\"}, {\"val_loss\": 13.93406541504129, \"val_crf_ner_loss\": 9.349364701848831, \"val_crf_term_loss\": 4.344773184644045, \"val_cls_rel_loss\": 0.23992731427624278, \"loss\": 12.419934537675646, \"crf_ner_loss\": 8.50323814815945, \"crf_term_loss\": 3.7242375877168445, \"cls_rel_loss\": 0.19245871818727917, \"f1_ner\": 0.44039530574428665, \"f1_term\": 0.4196141479099679, \"f1_rel\": 0.5310245310245311, \"epoch\": 13, \"method\": \"KPM*\"}, {\"val_loss\": 13.889922232523451, \"val_crf_ner_loss\": 9.325161039394184, \"val_crf_term_loss\": 4.326426059660251, \"val_cls_rel_loss\": 0.23833496096360424, \"loss\": 12.857303248511421, \"crf_ner_loss\": 8.788895103666517, \"crf_term_loss\": 3.8705077038870916, \"cls_rel_loss\": 0.19790057258473503, \"f1_ner\": 0.4642416769420469, \"f1_term\": 0.46090225563909765, \"f1_rel\": 0.5103503184713376, \"epoch\": 14, \"method\": \"KPM*\"}, {\"val_loss\": 13.87707072278879, \"val_crf_ner_loss\": 9.302793358364244, \"val_crf_term_loss\": 4.334407386988619, \"val_cls_rel_loss\": 0.23986962763932498, \"loss\": 11.917991108364529, \"crf_ner_loss\": 8.163964536454943, \"crf_term_loss\": 3.5885711246066623, \"cls_rel_loss\": 0.16545550607972676, \"f1_ner\": 0.4575202641849294, \"f1_term\": 0.46907817969661614, \"f1_rel\": 0.5349716446124764, \"epoch\": 15, \"method\": \"KPM*\"}, {\"val_loss\": 13.851704193727814, \"val_crf_ner_loss\": 9.281905269970858, \"val_crf_term_loss\": 4.333367421679253, \"val_cls_rel_loss\": 0.23643167219022765, \"loss\": 10.783254888322618, \"crf_ner_loss\": 7.348796447118123, \"crf_term_loss\": 3.2645195060306125, \"cls_rel_loss\": 0.16993894593583214, \"f1_ner\": 0.4778384942319369, \"f1_term\": 0.4589409056024559, \"f1_rel\": 0.5289183222958057, \"epoch\": 16, \"method\": \"KPM*\"}, {\"val_loss\": 13.83708848396357, \"val_crf_ner_loss\": 9.260113730047742, \"val_crf_term_loss\": 4.33385403486934, \"val_cls_rel_loss\": 0.24312052565769557, \"loss\": 13.709110021591187, \"crf_ner_loss\": 9.401966942681206, \"crf_term_loss\": 4.126997801992628, \"cls_rel_loss\": 0.18014505836698744, \"f1_ner\": 0.5148272699480281, \"f1_term\": 0.4684684684684685, \"f1_rel\": 0.526984126984127, \"epoch\": 17, \"method\": \"KPM*\"}, {\"val_loss\": 13.786796033817486, \"val_crf_ner_loss\": 9.23925695280089, \"val_crf_term_loss\": 4.309074413167299, \"val_cls_rel_loss\": 0.23846472701887145, \"loss\": 11.166400644514296, \"crf_ner_loss\": 7.633126152886285, \"crf_term_loss\": 3.3733496136135526, \"cls_rel_loss\": 0.15992481178707546, \"f1_ner\": 0.5514018691588785, \"f1_term\": 0.48030303030303034, \"f1_rel\": 0.5301629238221048, \"epoch\": 18, \"method\": \"KPM*\"}, {\"val_loss\": 13.805246923961779, \"val_crf_ner_loss\": 9.218766731067296, \"val_crf_term_loss\": 4.346495255936671, \"val_cls_rel_loss\": 0.23998485864513983, \"loss\": 11.829424328274197, \"crf_ner_loss\": 8.086160898208618, \"crf_term_loss\": 3.5691614283455744, \"cls_rel_loss\": 0.17410197026199764, \"f1_ner\": 0.548983924780103, \"f1_term\": 0.46465481603968584, \"f1_rel\": 0.5345402572653645, \"epoch\": 19, \"method\": \"KPM*\"}, {\"val_loss\": 13.806655104142907, \"val_crf_ner_loss\": 9.205200129181799, \"val_crf_term_loss\": 4.366030349348583, \"val_cls_rel_loss\": 0.23542456492020267, \"loss\": 12.593680832121107, \"crf_ner_loss\": 8.6165079275767, \"crf_term_loss\": 3.8064297437667847, \"cls_rel_loss\": 0.17074299852053323, \"f1_ner\": 0.5857970131057604, \"f1_term\": 0.45304347826086955, \"f1_rel\": 0.5360339142722562, \"epoch\": 20, \"method\": \"KPM*\"}, {\"val_loss\": 15.560061750620822, \"val_crf_ner_loss\": 10.219653251397348, \"val_crf_term_loss\": 4.834240241642416, \"val_cls_rel_loss\": 0.5061685951956867, \"loss\": 14.61849053700765, \"crf_ner_loss\": 10.024742497338188, \"crf_term_loss\": 4.184552060233222, \"cls_rel_loss\": 0.4091958983076943, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"KPM\"}, {\"val_loss\": 15.198520618633632, \"val_crf_ner_loss\": 10.098720501809225, \"val_crf_term_loss\": 4.648429468600419, \"val_cls_rel_loss\": 0.45137060819751157, \"loss\": 14.341292593214247, \"crf_ner_loss\": 9.808199167251587, \"crf_term_loss\": 4.236886435084873, \"cls_rel_loss\": 0.2962071845928828, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 2, \"method\": \"KPM\"}, {\"val_loss\": 14.928212701839252, \"val_crf_ner_loss\": 9.958176167341914, \"val_crf_term_loss\": 4.5732383902055505, \"val_cls_rel_loss\": 0.39679846711402394, \"loss\": 13.449421485265097, \"crf_ner_loss\": 9.17795599831475, \"crf_term_loss\": 3.9921832614474826, \"cls_rel_loss\": 0.2792823016643524, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.18351741716227699, \"epoch\": 3, \"method\": \"KPM\"}, {\"val_loss\": 14.76241027873798, \"val_crf_ner_loss\": 9.874341665393244, \"val_crf_term_loss\": 4.516334484963521, \"val_cls_rel_loss\": 0.3717341934242388, \"loss\": 13.711050351460775, \"crf_ner_loss\": 9.378151390287611, \"crf_term_loss\": 4.080903225474888, \"cls_rel_loss\": 0.2519957605335448, \"f1_ner\": 0.0, \"f1_term\": 0.026195153896529145, \"f1_rel\": 0.13408876298394712, \"epoch\": 4, \"method\": \"KPM\"}, {\"val_loss\": 14.577161350389467, \"val_crf_ner_loss\": 9.767085399070796, \"val_crf_term_loss\": 4.478528411719051, \"val_cls_rel_loss\": 0.3315476359677141, \"loss\": 13.33203911781311, \"crf_ner_loss\": 9.157547314961752, \"crf_term_loss\": 3.9552657339307995, \"cls_rel_loss\": 0.21922588100035986, \"f1_ner\": 0.0, \"f1_term\": 0.022696929238985312, \"f1_rel\": 0.3236009732360097, \"epoch\": 5, \"method\": \"KPM\"}, {\"val_loss\": 14.43750627893601, \"val_crf_ner_loss\": 9.673909403111812, \"val_crf_term_loss\": 4.4633757284958, \"val_cls_rel_loss\": 0.3002214940795063, \"loss\": 14.043683263990614, \"crf_ner_loss\": 9.638826052347818, \"crf_term_loss\": 4.180516984727648, \"cls_rel_loss\": 0.22434028486410776, \"f1_ner\": 0.0, \"f1_term\": 0.02976995940460081, \"f1_rel\": 0.4169435215946844, \"epoch\": 6, \"method\": \"KPM\"}, {\"val_loss\": 14.335392321983393, \"val_crf_ner_loss\": 9.601754004067748, \"val_crf_term_loss\": 4.454988237715115, \"val_cls_rel_loss\": 0.27864985352885113, \"loss\": 13.471335539550505, \"crf_ner_loss\": 9.256209160061468, \"crf_term_loss\": 4.0231675560608045, \"cls_rel_loss\": 0.19195865938072826, \"f1_ner\": 0.09342039176293321, \"f1_term\": 0.06107862248213125, \"f1_rel\": 0.4675767918088737, \"epoch\": 7, \"method\": \"KPM\"}, {\"val_loss\": 14.248672032878346, \"val_crf_ner_loss\": 9.550206334051424, \"val_crf_term_loss\": 4.43272578629264, \"val_cls_rel_loss\": 0.2657397724416134, \"loss\": 10.864235109753078, \"crf_ner_loss\": 7.41835814052158, \"crf_term_loss\": 3.244785295592414, \"cls_rel_loss\": 0.20109177629152933, \"f1_ner\": 0.23877245508982034, \"f1_term\": 0.09577114427860696, \"f1_rel\": 0.49005424954792043, \"epoch\": 8, \"method\": \"KPM\"}, {\"val_loss\": 14.169491339774027, \"val_crf_ner_loss\": 9.51067601503247, \"val_crf_term_loss\": 4.3926376635140745, \"val_cls_rel_loss\": 0.2661778715840221, \"loss\": 11.57474003897773, \"crf_ner_loss\": 7.915637572606404, \"crf_term_loss\": 3.47707642449273, \"cls_rel_loss\": 0.1820261784725719, \"f1_ner\": 0.27753743760399335, \"f1_term\": 0.1713106295149639, \"f1_rel\": 0.49172576832151305, \"epoch\": 9, \"method\": \"KPM\"}, {\"val_loss\": 14.134371740104509, \"val_crf_ner_loss\": 9.47610334758341, \"val_crf_term_loss\": 4.3935526531108104, \"val_cls_rel_loss\": 0.2647157968395818, \"loss\": 12.72979211807251, \"crf_ner_loss\": 8.73153993818495, \"crf_term_loss\": 3.82661145263248, \"cls_rel_loss\": 0.17164075954092872, \"f1_ner\": 0.33240826395312983, \"f1_term\": 0.1515837104072398, \"f1_rel\": 0.484920634920635, \"epoch\": 10, \"method\": \"KPM\"}, {\"val_loss\": 14.109744851606605, \"val_crf_ner_loss\": 9.441108084073031, \"val_crf_term_loss\": 4.409738464077024, \"val_cls_rel_loss\": 0.25889818859796454, \"loss\": 13.497358322143555, \"crf_ner_loss\": 9.285651869244045, \"crf_term_loss\": 4.041120608647664, \"cls_rel_loss\": 0.1705858620504538, \"f1_ner\": 0.35988926484158723, \"f1_term\": 0.19556566230812963, \"f1_rel\": 0.5080605764533463, \"epoch\": 11, \"method\": \"KPM\"}, {\"val_loss\": 14.076229311253902, \"val_crf_ner_loss\": 9.408538179676029, \"val_crf_term_loss\": 4.402429541532141, \"val_cls_rel_loss\": 0.26526164660488605, \"loss\": 13.330087608761257, \"crf_ner_loss\": 9.17171483569675, \"crf_term_loss\": 3.995752837922838, \"cls_rel_loss\": 0.16262009574307335, \"f1_ner\": 0.40727957326639475, \"f1_term\": 0.2764673614920461, \"f1_rel\": 0.5076190476190476, \"epoch\": 12, \"method\": \"KPM\"}, {\"val_loss\": 14.025057542063024, \"val_crf_ner_loss\": 9.383817965096801, \"val_crf_term_loss\": 4.3899574227576705, \"val_cls_rel_loss\": 0.2512820189016579, \"loss\": 10.935200108422173, \"crf_ner_loss\": 7.4790145556132, \"crf_term_loss\": 3.2752224736743503, \"cls_rel_loss\": 0.18096306009425056, \"f1_ner\": 0.43491767629698663, \"f1_term\": 0.33554987212276216, \"f1_rel\": 0.5099601593625498, \"epoch\": 13, \"method\": \"KPM\"}, {\"val_loss\": 13.95655904199085, \"val_crf_ner_loss\": 9.359742932076001, \"val_crf_term_loss\": 4.346859073116832, \"val_cls_rel_loss\": 0.24995723257969765, \"loss\": 13.672428343031141, \"crf_ner_loss\": 9.416304641299778, \"crf_term_loss\": 4.106093181504144, \"cls_rel_loss\": 0.15003018039796087, \"f1_ner\": 0.4515727187791965, \"f1_term\": 0.3736749116607773, \"f1_rel\": 0.5114813147230977, \"epoch\": 14, \"method\": \"KPM\"}, {\"val_loss\": 13.93107981577407, \"val_crf_ner_loss\": 9.336562083585418, \"val_crf_term_loss\": 4.343606795707759, \"val_cls_rel_loss\": 0.2509109221235679, \"loss\": 11.906590170330471, \"crf_ner_loss\": 8.182677851782906, \"crf_term_loss\": 3.573321567641364, \"cls_rel_loss\": 0.1505906908876366, \"f1_ner\": 0.47516198704103674, \"f1_term\": 0.40602836879432624, \"f1_rel\": 0.5275800711743772, \"epoch\": 15, \"method\": \"KPM\"}, {\"val_loss\": 13.928269602086422, \"val_crf_ner_loss\": 9.317142190724393, \"val_crf_term_loss\": 4.366590955831708, \"val_cls_rel_loss\": 0.2445363311001854, \"loss\": 12.97074524561564, \"crf_ner_loss\": 8.915461487240261, \"crf_term_loss\": 3.8990012539757624, \"cls_rel_loss\": 0.15628258718384636, \"f1_ner\": 0.4880147510755992, \"f1_term\": 0.40456621004566207, \"f1_rel\": 0.5264562027567808, \"epoch\": 16, \"method\": \"KPM\"}, {\"val_loss\": 13.899771871357938, \"val_crf_ner_loss\": 9.292585501705643, \"val_crf_term_loss\": 4.348387460639007, \"val_cls_rel_loss\": 0.2587989359876535, \"loss\": 13.465459373262194, \"crf_ner_loss\": 9.277238872316149, \"crf_term_loss\": 4.026697039604187, \"cls_rel_loss\": 0.16152334958314896, \"f1_ner\": 0.5033313143549364, \"f1_term\": 0.37316176470588236, \"f1_rel\": 0.5272895467160038, \"epoch\": 17, \"method\": \"KPM\"}, {\"val_loss\": 13.900357114137524, \"val_crf_ner_loss\": 9.276182639337804, \"val_crf_term_loss\": 4.360796211409743, \"val_cls_rel_loss\": 0.26337817289533405, \"loss\": 11.405402104059855, \"crf_ner_loss\": 7.84924070040385, \"crf_term_loss\": 3.416629751523336, \"cls_rel_loss\": 0.13953178748488426, \"f1_ner\": 0.5406713032960387, \"f1_term\": 0.4014665444546288, \"f1_rel\": 0.5261277355962483, \"epoch\": 18, \"method\": \"KPM\"}, {\"val_loss\": 13.896165350057784, \"val_crf_ner_loss\": 9.257388033135964, \"val_crf_term_loss\": 4.384082302559901, \"val_cls_rel_loss\": 0.2546948533858696, \"loss\": 11.414863692389595, \"crf_ner_loss\": 7.842244174745348, \"crf_term_loss\": 3.435810407002767, \"cls_rel_loss\": 0.13680922778116333, \"f1_ner\": 0.5773386847792529, \"f1_term\": 0.3829145728643216, \"f1_rel\": 0.5267175572519084, \"epoch\": 19, \"method\": \"KPM\"}, {\"val_loss\": 13.877817710820775, \"val_crf_ner_loss\": 9.246109692719731, \"val_crf_term_loss\": 4.375795293028337, \"val_cls_rel_loss\": 0.255912614564826, \"loss\": 13.860608206854927, \"crf_ner_loss\": 9.544105344348484, \"crf_term_loss\": 4.17823080221812, \"cls_rel_loss\": 0.13827224034402105, \"f1_ner\": 0.5676408556794216, \"f1_term\": 0.3892750744786495, \"f1_rel\": 0.5167381974248927, \"epoch\": 20, \"method\": \"KPM\"}, {\"val_loss\": 15.243806299501962, \"val_crf_ner_loss\": 10.107058149184624, \"val_crf_term_loss\": 4.683029660343253, \"val_cls_rel_loss\": 0.4537181549698767, \"loss\": 16.699586031248245, \"crf_ner_loss\": 11.24831004794553, \"crf_term_loss\": 4.924503230362487, \"cls_rel_loss\": 0.5267729591980255, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"Manual\"}, {\"val_loss\": 14.912682700331194, \"val_crf_ner_loss\": 9.9304054705766, \"val_crf_term_loss\": 4.586394315218404, \"val_cls_rel_loss\": 0.39588289452295233, \"loss\": 14.990882839230325, \"crf_ner_loss\": 9.938884138203353, \"crf_term_loss\": 4.628164531515656, \"cls_rel_loss\": 0.4238341283455169, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.1015625, \"epoch\": 2, \"method\": \"Manual\"}, {\"val_loss\": 14.627378710865104, \"val_crf_ner_loss\": 9.815573285095883, \"val_crf_term_loss\": 4.461484249490891, \"val_cls_rel_loss\": 0.3503211245049525, \"loss\": 14.700915823737494, \"crf_ner_loss\": 9.804691932184234, \"crf_term_loss\": 4.521886180630691, \"cls_rel_loss\": 0.374337677689765, \"f1_ner\": 0.0, \"f1_term\": 0.05017103762827822, \"f1_rel\": 0.25273010920436817, \"epoch\": 3, \"method\": \"Manual\"}, {\"val_loss\": 14.398991758805993, \"val_crf_ner_loss\": 9.709294179930303, \"val_crf_term_loss\": 4.380614814096994, \"val_cls_rel_loss\": 0.3090829792684012, \"loss\": 14.475410674115738, \"crf_ner_loss\": 9.713599438289945, \"crf_term_loss\": 4.423988352576606, \"cls_rel_loss\": 0.33782292098450145, \"f1_ner\": 0.0, \"f1_term\": 0.14696223316912974, \"f1_rel\": 0.3793103448275862, \"epoch\": 4, \"method\": \"Manual\"}, {\"val_loss\": 14.195851969892962, \"val_crf_ner_loss\": 9.582875979207728, \"val_crf_term_loss\": 4.323448491792609, \"val_cls_rel_loss\": 0.2895272591688337, \"loss\": 14.247947795785588, \"crf_ner_loss\": 9.613109622927878, \"crf_term_loss\": 4.350397923009859, \"cls_rel_loss\": 0.284440088615143, \"f1_ner\": 0.0, \"f1_term\": 0.26815240833932424, \"f1_rel\": 0.4473684210526315, \"epoch\": 5, \"method\": \"Manual\"}, {\"val_loss\": 14.019642192951954, \"val_crf_ner_loss\": 9.483940947664916, \"val_crf_term_loss\": 4.283573719706848, \"val_cls_rel_loss\": 0.2521274812030096, \"loss\": 14.042691244495858, \"crf_ner_loss\": 9.504135097531107, \"crf_term_loss\": 4.2868207581609274, \"cls_rel_loss\": 0.2517352447235327, \"f1_ner\": 0.0460122699386503, \"f1_term\": 0.3209130580731789, \"f1_rel\": 0.5104602510460251, \"epoch\": 6, \"method\": \"Manual\"}, {\"val_loss\": 13.945621532245275, \"val_crf_ner_loss\": 9.416973201027751, \"val_crf_term_loss\": 4.291519514835664, \"val_cls_rel_loss\": 0.23712905654071892, \"loss\": 13.894049123036776, \"crf_ner_loss\": 9.421672182975055, \"crf_term_loss\": 4.242246555767471, \"cls_rel_loss\": 0.23013035835122034, \"f1_ner\": 0.14721873693015475, \"f1_term\": 0.4274809160305343, \"f1_rel\": 0.5217391304347826, \"epoch\": 7, \"method\": \"Manual\"}, {\"val_loss\": 13.847039943193867, \"val_crf_ner_loss\": 9.365032241292244, \"val_crf_term_loss\": 4.256806112553951, \"val_cls_rel_loss\": 0.2252013226495172, \"loss\": 13.77873220375116, \"crf_ner_loss\": 9.358538778565771, \"crf_term_loss\": 4.205774499358033, \"cls_rel_loss\": 0.21441889816908527, \"f1_ner\": 0.2721679334057184, \"f1_term\": 0.4627766599597586, \"f1_rel\": 0.5461254612546126, \"epoch\": 8, \"method\": \"Manual\"}, {\"val_loss\": 13.772800000044551, \"val_crf_ner_loss\": 9.32188389771176, \"val_crf_term_loss\": 4.203255200908131, \"val_cls_rel_loss\": 0.24766094710705053, \"loss\": 13.690944232529016, \"crf_ner_loss\": 9.309969462936731, \"crf_term_loss\": 4.179576609632094, \"cls_rel_loss\": 0.20139821935043062, \"f1_ner\": 0.34415186772810774, \"f1_term\": 0.4714754098360656, \"f1_rel\": 0.5033647375504711, \"epoch\": 9, \"method\": \"Manual\"}, {\"val_loss\": 13.694452710395312, \"val_crf_ner_loss\": 9.28409958755883, \"val_crf_term_loss\": 4.189713124811214, \"val_cls_rel_loss\": 0.22063998192766288, \"loss\": 13.6023733564418, \"crf_ner_loss\": 9.266674076052878, \"crf_term_loss\": 4.143799555387428, \"cls_rel_loss\": 0.1918996330216634, \"f1_ner\": 0.3637484586929716, \"f1_term\": 0.4625719769673704, \"f1_rel\": 0.5594906486271388, \"epoch\": 10, \"method\": \"Manual\"}, {\"val_loss\": 13.650851120913986, \"val_crf_ner_loss\": 9.250646060400635, \"val_crf_term_loss\": 4.17821693246382, \"val_cls_rel_loss\": 0.22198789789728876, \"loss\": 13.528535136215979, \"crf_ner_loss\": 9.231000474888644, \"crf_term_loss\": 4.122180232041174, \"cls_rel_loss\": 0.1753543517786822, \"f1_ner\": 0.39359408684939945, \"f1_term\": 0.5359349593495935, \"f1_rel\": 0.5620703598867771, \"epoch\": 11, \"method\": \"Manual\"}, {\"val_loss\": 13.651739012585939, \"val_crf_ner_loss\": 9.219236165067576, \"val_crf_term_loss\": 4.200671663249496, \"val_cls_rel_loss\": 0.23183101219852476, \"loss\": 13.46437355425718, \"crf_ner_loss\": 9.193118143424714, \"crf_term_loss\": 4.103265326657741, \"cls_rel_loss\": 0.16799015675088486, \"f1_ner\": 0.4190246839253462, \"f1_term\": 0.5431290898274838, \"f1_rel\": 0.554696132596685, \"epoch\": 12, \"method\": \"Manual\"}, {\"val_loss\": 13.602622683030846, \"val_crf_ner_loss\": 9.189025001804325, \"val_crf_term_loss\": 4.1845643494251, \"val_cls_rel_loss\": 0.22903330939529587, \"loss\": 13.393721086515797, \"crf_ner_loss\": 9.160476073944311, \"crf_term_loss\": 4.07530150653647, \"cls_rel_loss\": 0.15794322977391936, \"f1_ner\": 0.4428442844284428, \"f1_term\": 0.5249169435215947, \"f1_rel\": 0.5605665300037271, \"epoch\": 13, \"method\": \"Manual\"}, {\"val_loss\": 13.545565184015427, \"val_crf_ner_loss\": 9.161734525304642, \"val_crf_term_loss\": 4.164743535710077, \"val_cls_rel_loss\": 0.21908710266116763, \"loss\": 13.347055352849068, \"crf_ner_loss\": 9.13083744735169, \"crf_term_loss\": 4.064087275978473, \"cls_rel_loss\": 0.15213072224915455, \"f1_ner\": 0.4908709967075726, \"f1_term\": 0.5755251750583528, \"f1_rel\": 0.5927209705372617, \"epoch\": 14, \"method\": \"Manual\"}, {\"val_loss\": 13.566906312956426, \"val_crf_ner_loss\": 9.141436623830865, \"val_crf_term_loss\": 4.179225100217945, \"val_cls_rel_loss\": 0.24624451755607216, \"loss\": 13.293653714571068, \"crf_ner_loss\": 9.101718058689036, \"crf_term_loss\": 4.0504371982684235, \"cls_rel_loss\": 0.14149870814608156, \"f1_ner\": 0.5365555215662282, \"f1_term\": 0.5604080331526937, \"f1_rel\": 0.5522174535050072, \"epoch\": 15, \"method\": \"Manual\"}, {\"val_loss\": 13.528757687032657, \"val_crf_ner_loss\": 9.11671873078729, \"val_crf_term_loss\": 4.181043214171472, \"val_cls_rel_loss\": 0.23099547298285214, \"loss\": 13.262428640461653, \"crf_ner_loss\": 9.074218688251303, \"crf_term_loss\": 4.034358688395658, \"cls_rel_loss\": 0.15385115746971514, \"f1_ner\": 0.5820622330689443, \"f1_term\": 0.5869017632241814, \"f1_rel\": 0.5668409260642271, \"epoch\": 16, \"method\": \"Manual\"}, {\"val_loss\": 13.499076745805949, \"val_crf_ner_loss\": 9.098606285387582, \"val_crf_term_loss\": 4.168222627500548, \"val_cls_rel_loss\": 0.23224780355056707, \"loss\": 13.205760331462614, \"crf_ner_loss\": 9.047224957308323, \"crf_term_loss\": 4.024408325016927, \"cls_rel_loss\": 0.1341272007218368, \"f1_ner\": 0.6072849709213346, \"f1_term\": 0.5895424836601307, \"f1_rel\": 0.6030061892130858, \"epoch\": 17, \"method\": \"Manual\"}, {\"val_loss\": 13.490994383818911, \"val_crf_ner_loss\": 9.081056586147225, \"val_crf_term_loss\": 4.172767678316492, \"val_cls_rel_loss\": 0.23717018896645872, \"loss\": 13.162245880785607, \"crf_ner_loss\": 9.026262454849354, \"crf_term_loss\": 4.005062815096738, \"cls_rel_loss\": 0.13092073208565336, \"f1_ner\": 0.61817065684469, \"f1_term\": 0.5970826580226904, \"f1_rel\": 0.6050000000000001, \"epoch\": 18, \"method\": \"Manual\"}, {\"val_loss\": 13.477757064095378, \"val_crf_ner_loss\": 9.067503215622727, \"val_crf_term_loss\": 4.169223362511962, \"val_cls_rel_loss\": 0.24103034129978096, \"loss\": 13.123809293019686, \"crf_ner_loss\": 9.004541664672413, \"crf_term_loss\": 4.000660311403892, \"cls_rel_loss\": 0.11860741154967452, \"f1_ner\": 0.6316436251920123, \"f1_term\": 0.6051834399192191, \"f1_rel\": 0.5977108944467996, \"epoch\": 19, \"method\": \"Manual\"}, {\"val_loss\": 13.485108493888465, \"val_crf_ner_loss\": 9.052033807239393, \"val_crf_term_loss\": 4.186074289962323, \"val_cls_rel_loss\": 0.247000602475048, \"loss\": 13.101578980041065, \"crf_ner_loss\": 8.987195536387052, \"crf_term_loss\": 4.005926411786525, \"cls_rel_loss\": 0.10845713746204651, \"f1_ner\": 0.6299357208448118, \"f1_term\": 0.6049889485317335, \"f1_rel\": 0.5819032761310452, \"epoch\": 20, \"method\": \"Manual\"}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.HConcatChart(...)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "te_results_df = results_df[results_df[\"method\"].isin([\"KPM*\", \"KPM\", \"Manual\"])]\n",
    "\n",
    "ner_loss = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"val_crf_ner_loss:Q\", title=\"NER Loss\", scale=alt.Scale(domain=[9, 10.4])),\n",
    "    color=alt.Color(\"method:N\", title=\"TE Method\")\n",
    ")\n",
    "\n",
    "term_loss = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"val_crf_term_loss:Q\", title=\"TE Loss\", scale=alt.Scale(domain=[4, 5])),\n",
    "    color=alt.Color(\"method:N\", title=\"TE Method\")\n",
    ")\n",
    "\n",
    "rel_loss = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"val_cls_rel_loss:Q\", title=\"RE Loss\", scale=alt.Scale(domain=[0, 0.6])),\n",
    "    color=alt.Color(\"method:N\", title=\"TE Method\")\n",
    ")\n",
    "\n",
    "all_charts = ner_loss | term_loss | rel_loss\n",
    "\n",
    "all_charts.configure_axis(\n",
    "    titleFont=\"Times New Roman\",\n",
    "    titleFontSize=18,\n",
    "    labelFont=\"Times New Roman\",\n",
    "    labelFontSize=18\n",
    ").configure_legend(\n",
    "    titleFont=\"Times New Roman\",\n",
    "    titleFontSize=20,\n",
    "    labelFont=\"Times New Roman\",\n",
    "    labelFontSize=18\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div id=\"altair-viz-40ac27e741e44769b9df9ae8cb322a4e\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-40ac27e741e44769b9df9ae8cb322a4e\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-40ac27e741e44769b9df9ae8cb322a4e\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm//vega-lite@4.8.1?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm//vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function loadScript(lib) {\n",
       "      return new Promise(function(resolve, reject) {\n",
       "        var s = document.createElement('script');\n",
       "        s.src = paths[lib];\n",
       "        s.async = true;\n",
       "        s.onload = () => resolve(paths[lib]);\n",
       "        s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else if (typeof vegaEmbed === \"function\") {\n",
       "      displayChart(vegaEmbed);\n",
       "    } else {\n",
       "      loadScript(\"vega\")\n",
       "        .then(() => loadScript(\"vega-lite\"))\n",
       "        .then(() => loadScript(\"vega-embed\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}, \"axis\": {\"labelFontSize\": 18, \"titleFontSize\": 18}, \"header\": {\"labelFontSize\": 20, \"titleFontSize\": 22}, \"legend\": {\"labelFontSize\": 18, \"titleFontSize\": 20}, \"title\": {\"fontSize\": 24}}, \"hconcat\": [{\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"TE Method\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"f1_ner\", \"scale\": {\"domain\": [0, 0.65]}, \"title\": \"NER F1\"}}}, {\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"TE Method\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"f1_term\", \"scale\": {\"domain\": [0, 0.65]}, \"title\": \"Terms F1\"}}}, {\"mark\": \"line\", \"encoding\": {\"color\": {\"type\": \"nominal\", \"field\": \"method\", \"title\": \"TE Method\"}, \"x\": {\"type\": \"quantitative\", \"field\": \"epoch\"}, \"y\": {\"type\": \"quantitative\", \"field\": \"f1_rel\", \"scale\": {\"domain\": [0, 0.65]}, \"title\": \"Rel F1\"}}}], \"data\": {\"name\": \"data-d179cc84f596b886aeafa9ac8a80335d\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v4.8.1.json\", \"datasets\": {\"data-d179cc84f596b886aeafa9ac8a80335d\": [{\"val_loss\": 15.301897678932134, \"val_crf_ner_loss\": 10.182450364105893, \"val_crf_term_loss\": 4.664121624327054, \"val_cls_rel_loss\": 0.4553257749028449, \"loss\": 13.774118158552382, \"crf_ner_loss\": 9.44224182764689, \"crf_term_loss\": 3.892859763569302, \"cls_rel_loss\": 0.43901652925544316, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"KPM*\"}, {\"val_loss\": 15.088710158410734, \"val_crf_ner_loss\": 10.042704025324245, \"val_crf_term_loss\": 4.624338203973144, \"val_cls_rel_loss\": 0.4216679147560231, \"loss\": 14.391434563530815, \"crf_ner_loss\": 9.721958584255642, \"crf_term_loss\": 4.309103541904026, \"cls_rel_loss\": 0.3603723694880803, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 2, \"method\": \"KPM*\"}, {\"val_loss\": 14.868212205650163, \"val_crf_ner_loss\": 9.94457231994963, \"val_crf_term_loss\": 4.530394581112549, \"val_cls_rel_loss\": 0.3932453250362925, \"loss\": 12.414401796129015, \"crf_ner_loss\": 8.338009569380018, \"crf_term_loss\": 3.7816480265723333, \"cls_rel_loss\": 0.29474418196413255, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.021929824561403508, \"epoch\": 3, \"method\": \"KPM*\"}, {\"val_loss\": 14.644684871617894, \"val_crf_ner_loss\": 9.838034796888811, \"val_crf_term_loss\": 4.464446843975652, \"val_cls_rel_loss\": 0.3422032877476546, \"loss\": 12.452283435397678, \"crf_ner_loss\": 8.426005283991495, \"crf_term_loss\": 3.7280285490883722, \"cls_rel_loss\": 0.29824961970249814, \"f1_ner\": 0.0, \"f1_term\": 0.012371134020618556, \"f1_rel\": 0.33200531208499334, \"epoch\": 4, \"method\": \"KPM*\"}, {\"val_loss\": 14.48356381646038, \"val_crf_ner_loss\": 9.733082833951407, \"val_crf_term_loss\": 4.437290826853174, \"val_cls_rel_loss\": 0.3131902344035406, \"loss\": 12.599537902408176, \"crf_ner_loss\": 8.55139226383633, \"crf_term_loss\": 3.790961265563965, \"cls_rel_loss\": 0.2571844566199515, \"f1_ner\": 0.0, \"f1_term\": 0.031862745098039214, \"f1_rel\": 0.33602347762289064, \"epoch\": 5, \"method\": \"KPM*\"}, {\"val_loss\": 14.285567158330096, \"val_crf_ner_loss\": 9.62895664333427, \"val_crf_term_loss\": 4.376269838235674, \"val_cls_rel_loss\": 0.28034074550127464, \"loss\": 11.80238119761149, \"crf_ner_loss\": 8.021390702989367, \"crf_term_loss\": 3.538855208290948, \"cls_rel_loss\": 0.24213523831632403, \"f1_ner\": 0.03489640130861505, \"f1_term\": 0.1641949152542373, \"f1_rel\": 0.4579710144927536, \"epoch\": 6, \"method\": \"KPM*\"}, {\"val_loss\": 14.190792644110909, \"val_crf_ner_loss\": 9.552197621686615, \"val_crf_term_loss\": 4.374607987647509, \"val_cls_rel_loss\": 0.2639873599919089, \"loss\": 15.422736909654406, \"crf_ner_loss\": 10.54382954703437, \"crf_term_loss\": 4.651935312483046, \"cls_rel_loss\": 0.22697221653329003, \"f1_ner\": 0.2111237230419977, \"f1_term\": 0.24483917426788285, \"f1_rel\": 0.4822411205602801, \"epoch\": 7, \"method\": \"KPM*\"}, {\"val_loss\": 14.100167079563558, \"val_crf_ner_loss\": 9.497426730872942, \"val_crf_term_loss\": 4.346293905355634, \"val_cls_rel_loss\": 0.2564464663937144, \"loss\": 13.520169470045301, \"crf_ner_loss\": 9.212770409054226, \"crf_term_loss\": 4.079206003083123, \"cls_rel_loss\": 0.2281930512852139, \"f1_ner\": 0.2951269732326699, \"f1_term\": 0.34651447207501024, \"f1_rel\": 0.48416112631990615, \"epoch\": 8, \"method\": \"KPM*\"}, {\"val_loss\": 14.051934238767972, \"val_crf_ner_loss\": 9.458889895111975, \"val_crf_term_loss\": 4.336595517875504, \"val_cls_rel_loss\": 0.25644875138345424, \"loss\": 12.486466672685411, \"crf_ner_loss\": 8.513386329015097, \"crf_term_loss\": 3.770048565334744, \"cls_rel_loss\": 0.20303168065018123, \"f1_ner\": 0.34801624492346145, \"f1_term\": 0.3886328725038403, \"f1_rel\": 0.5049739459971578, \"epoch\": 9, \"method\": \"KPM*\"}, {\"val_loss\": 14.029026644073264, \"val_crf_ner_loss\": 9.425868076129552, \"val_crf_term_loss\": 4.339041190425845, \"val_cls_rel_loss\": 0.2641175667734912, \"loss\": 11.872677952105645, \"crf_ner_loss\": 8.10128795533991, \"crf_term_loss\": 3.5802190727681955, \"cls_rel_loss\": 0.19117092780455594, \"f1_ner\": 0.4203262233375157, \"f1_term\": 0.4071246819338422, \"f1_rel\": 0.4720161834120027, \"epoch\": 10, \"method\": \"KPM*\"}, {\"val_loss\": 13.972422105552507, \"val_crf_ner_loss\": 9.398135232229302, \"val_crf_term_loss\": 4.321064592277916, \"val_cls_rel_loss\": 0.2532223985142951, \"loss\": 12.673591242896187, \"crf_ner_loss\": 8.674006409115261, \"crf_term_loss\": 3.8085707426071167, \"cls_rel_loss\": 0.19101394216219583, \"f1_ner\": 0.4384251968503937, \"f1_term\": 0.454308093994778, \"f1_rel\": 0.5206316861946001, \"epoch\": 11, \"method\": \"KPM*\"}, {\"val_loss\": 13.9664681636504, \"val_crf_ner_loss\": 9.375703832528886, \"val_crf_term_loss\": 4.348035911573981, \"val_cls_rel_loss\": 0.2427283042538775, \"loss\": 11.879486507839626, \"crf_ner_loss\": 8.127701388465034, \"crf_term_loss\": 3.56903256310357, \"cls_rel_loss\": 0.18275263408819833, \"f1_ner\": 0.41350210970464135, \"f1_term\": 0.44039603960396045, \"f1_rel\": 0.5142371440713982, \"epoch\": 12, \"method\": \"KPM*\"}, {\"val_loss\": 13.93406541504129, \"val_crf_ner_loss\": 9.349364701848831, \"val_crf_term_loss\": 4.344773184644045, \"val_cls_rel_loss\": 0.23992731427624278, \"loss\": 12.419934537675646, \"crf_ner_loss\": 8.50323814815945, \"crf_term_loss\": 3.7242375877168445, \"cls_rel_loss\": 0.19245871818727917, \"f1_ner\": 0.44039530574428665, \"f1_term\": 0.4196141479099679, \"f1_rel\": 0.5310245310245311, \"epoch\": 13, \"method\": \"KPM*\"}, {\"val_loss\": 13.889922232523451, \"val_crf_ner_loss\": 9.325161039394184, \"val_crf_term_loss\": 4.326426059660251, \"val_cls_rel_loss\": 0.23833496096360424, \"loss\": 12.857303248511421, \"crf_ner_loss\": 8.788895103666517, \"crf_term_loss\": 3.8705077038870916, \"cls_rel_loss\": 0.19790057258473503, \"f1_ner\": 0.4642416769420469, \"f1_term\": 0.46090225563909765, \"f1_rel\": 0.5103503184713376, \"epoch\": 14, \"method\": \"KPM*\"}, {\"val_loss\": 13.87707072278879, \"val_crf_ner_loss\": 9.302793358364244, \"val_crf_term_loss\": 4.334407386988619, \"val_cls_rel_loss\": 0.23986962763932498, \"loss\": 11.917991108364529, \"crf_ner_loss\": 8.163964536454943, \"crf_term_loss\": 3.5885711246066623, \"cls_rel_loss\": 0.16545550607972676, \"f1_ner\": 0.4575202641849294, \"f1_term\": 0.46907817969661614, \"f1_rel\": 0.5349716446124764, \"epoch\": 15, \"method\": \"KPM*\"}, {\"val_loss\": 13.851704193727814, \"val_crf_ner_loss\": 9.281905269970858, \"val_crf_term_loss\": 4.333367421679253, \"val_cls_rel_loss\": 0.23643167219022765, \"loss\": 10.783254888322618, \"crf_ner_loss\": 7.348796447118123, \"crf_term_loss\": 3.2645195060306125, \"cls_rel_loss\": 0.16993894593583214, \"f1_ner\": 0.4778384942319369, \"f1_term\": 0.4589409056024559, \"f1_rel\": 0.5289183222958057, \"epoch\": 16, \"method\": \"KPM*\"}, {\"val_loss\": 13.83708848396357, \"val_crf_ner_loss\": 9.260113730047742, \"val_crf_term_loss\": 4.33385403486934, \"val_cls_rel_loss\": 0.24312052565769557, \"loss\": 13.709110021591187, \"crf_ner_loss\": 9.401966942681206, \"crf_term_loss\": 4.126997801992628, \"cls_rel_loss\": 0.18014505836698744, \"f1_ner\": 0.5148272699480281, \"f1_term\": 0.4684684684684685, \"f1_rel\": 0.526984126984127, \"epoch\": 17, \"method\": \"KPM*\"}, {\"val_loss\": 13.786796033817486, \"val_crf_ner_loss\": 9.23925695280089, \"val_crf_term_loss\": 4.309074413167299, \"val_cls_rel_loss\": 0.23846472701887145, \"loss\": 11.166400644514296, \"crf_ner_loss\": 7.633126152886285, \"crf_term_loss\": 3.3733496136135526, \"cls_rel_loss\": 0.15992481178707546, \"f1_ner\": 0.5514018691588785, \"f1_term\": 0.48030303030303034, \"f1_rel\": 0.5301629238221048, \"epoch\": 18, \"method\": \"KPM*\"}, {\"val_loss\": 13.805246923961779, \"val_crf_ner_loss\": 9.218766731067296, \"val_crf_term_loss\": 4.346495255936671, \"val_cls_rel_loss\": 0.23998485864513983, \"loss\": 11.829424328274197, \"crf_ner_loss\": 8.086160898208618, \"crf_term_loss\": 3.5691614283455744, \"cls_rel_loss\": 0.17410197026199764, \"f1_ner\": 0.548983924780103, \"f1_term\": 0.46465481603968584, \"f1_rel\": 0.5345402572653645, \"epoch\": 19, \"method\": \"KPM*\"}, {\"val_loss\": 13.806655104142907, \"val_crf_ner_loss\": 9.205200129181799, \"val_crf_term_loss\": 4.366030349348583, \"val_cls_rel_loss\": 0.23542456492020267, \"loss\": 12.593680832121107, \"crf_ner_loss\": 8.6165079275767, \"crf_term_loss\": 3.8064297437667847, \"cls_rel_loss\": 0.17074299852053323, \"f1_ner\": 0.5857970131057604, \"f1_term\": 0.45304347826086955, \"f1_rel\": 0.5360339142722562, \"epoch\": 20, \"method\": \"KPM*\"}, {\"val_loss\": 15.560061750620822, \"val_crf_ner_loss\": 10.219653251397348, \"val_crf_term_loss\": 4.834240241642416, \"val_cls_rel_loss\": 0.5061685951956867, \"loss\": 14.61849053700765, \"crf_ner_loss\": 10.024742497338188, \"crf_term_loss\": 4.184552060233222, \"cls_rel_loss\": 0.4091958983076943, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"KPM\"}, {\"val_loss\": 15.198520618633632, \"val_crf_ner_loss\": 10.098720501809225, \"val_crf_term_loss\": 4.648429468600419, \"val_cls_rel_loss\": 0.45137060819751157, \"loss\": 14.341292593214247, \"crf_ner_loss\": 9.808199167251587, \"crf_term_loss\": 4.236886435084873, \"cls_rel_loss\": 0.2962071845928828, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 2, \"method\": \"KPM\"}, {\"val_loss\": 14.928212701839252, \"val_crf_ner_loss\": 9.958176167341914, \"val_crf_term_loss\": 4.5732383902055505, \"val_cls_rel_loss\": 0.39679846711402394, \"loss\": 13.449421485265097, \"crf_ner_loss\": 9.17795599831475, \"crf_term_loss\": 3.9921832614474826, \"cls_rel_loss\": 0.2792823016643524, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.18351741716227699, \"epoch\": 3, \"method\": \"KPM\"}, {\"val_loss\": 14.76241027873798, \"val_crf_ner_loss\": 9.874341665393244, \"val_crf_term_loss\": 4.516334484963521, \"val_cls_rel_loss\": 0.3717341934242388, \"loss\": 13.711050351460775, \"crf_ner_loss\": 9.378151390287611, \"crf_term_loss\": 4.080903225474888, \"cls_rel_loss\": 0.2519957605335448, \"f1_ner\": 0.0, \"f1_term\": 0.026195153896529145, \"f1_rel\": 0.13408876298394712, \"epoch\": 4, \"method\": \"KPM\"}, {\"val_loss\": 14.577161350389467, \"val_crf_ner_loss\": 9.767085399070796, \"val_crf_term_loss\": 4.478528411719051, \"val_cls_rel_loss\": 0.3315476359677141, \"loss\": 13.33203911781311, \"crf_ner_loss\": 9.157547314961752, \"crf_term_loss\": 3.9552657339307995, \"cls_rel_loss\": 0.21922588100035986, \"f1_ner\": 0.0, \"f1_term\": 0.022696929238985312, \"f1_rel\": 0.3236009732360097, \"epoch\": 5, \"method\": \"KPM\"}, {\"val_loss\": 14.43750627893601, \"val_crf_ner_loss\": 9.673909403111812, \"val_crf_term_loss\": 4.4633757284958, \"val_cls_rel_loss\": 0.3002214940795063, \"loss\": 14.043683263990614, \"crf_ner_loss\": 9.638826052347818, \"crf_term_loss\": 4.180516984727648, \"cls_rel_loss\": 0.22434028486410776, \"f1_ner\": 0.0, \"f1_term\": 0.02976995940460081, \"f1_rel\": 0.4169435215946844, \"epoch\": 6, \"method\": \"KPM\"}, {\"val_loss\": 14.335392321983393, \"val_crf_ner_loss\": 9.601754004067748, \"val_crf_term_loss\": 4.454988237715115, \"val_cls_rel_loss\": 0.27864985352885113, \"loss\": 13.471335539550505, \"crf_ner_loss\": 9.256209160061468, \"crf_term_loss\": 4.0231675560608045, \"cls_rel_loss\": 0.19195865938072826, \"f1_ner\": 0.09342039176293321, \"f1_term\": 0.06107862248213125, \"f1_rel\": 0.4675767918088737, \"epoch\": 7, \"method\": \"KPM\"}, {\"val_loss\": 14.248672032878346, \"val_crf_ner_loss\": 9.550206334051424, \"val_crf_term_loss\": 4.43272578629264, \"val_cls_rel_loss\": 0.2657397724416134, \"loss\": 10.864235109753078, \"crf_ner_loss\": 7.41835814052158, \"crf_term_loss\": 3.244785295592414, \"cls_rel_loss\": 0.20109177629152933, \"f1_ner\": 0.23877245508982034, \"f1_term\": 0.09577114427860696, \"f1_rel\": 0.49005424954792043, \"epoch\": 8, \"method\": \"KPM\"}, {\"val_loss\": 14.169491339774027, \"val_crf_ner_loss\": 9.51067601503247, \"val_crf_term_loss\": 4.3926376635140745, \"val_cls_rel_loss\": 0.2661778715840221, \"loss\": 11.57474003897773, \"crf_ner_loss\": 7.915637572606404, \"crf_term_loss\": 3.47707642449273, \"cls_rel_loss\": 0.1820261784725719, \"f1_ner\": 0.27753743760399335, \"f1_term\": 0.1713106295149639, \"f1_rel\": 0.49172576832151305, \"epoch\": 9, \"method\": \"KPM\"}, {\"val_loss\": 14.134371740104509, \"val_crf_ner_loss\": 9.47610334758341, \"val_crf_term_loss\": 4.3935526531108104, \"val_cls_rel_loss\": 0.2647157968395818, \"loss\": 12.72979211807251, \"crf_ner_loss\": 8.73153993818495, \"crf_term_loss\": 3.82661145263248, \"cls_rel_loss\": 0.17164075954092872, \"f1_ner\": 0.33240826395312983, \"f1_term\": 0.1515837104072398, \"f1_rel\": 0.484920634920635, \"epoch\": 10, \"method\": \"KPM\"}, {\"val_loss\": 14.109744851606605, \"val_crf_ner_loss\": 9.441108084073031, \"val_crf_term_loss\": 4.409738464077024, \"val_cls_rel_loss\": 0.25889818859796454, \"loss\": 13.497358322143555, \"crf_ner_loss\": 9.285651869244045, \"crf_term_loss\": 4.041120608647664, \"cls_rel_loss\": 0.1705858620504538, \"f1_ner\": 0.35988926484158723, \"f1_term\": 0.19556566230812963, \"f1_rel\": 0.5080605764533463, \"epoch\": 11, \"method\": \"KPM\"}, {\"val_loss\": 14.076229311253902, \"val_crf_ner_loss\": 9.408538179676029, \"val_crf_term_loss\": 4.402429541532141, \"val_cls_rel_loss\": 0.26526164660488605, \"loss\": 13.330087608761257, \"crf_ner_loss\": 9.17171483569675, \"crf_term_loss\": 3.995752837922838, \"cls_rel_loss\": 0.16262009574307335, \"f1_ner\": 0.40727957326639475, \"f1_term\": 0.2764673614920461, \"f1_rel\": 0.5076190476190476, \"epoch\": 12, \"method\": \"KPM\"}, {\"val_loss\": 14.025057542063024, \"val_crf_ner_loss\": 9.383817965096801, \"val_crf_term_loss\": 4.3899574227576705, \"val_cls_rel_loss\": 0.2512820189016579, \"loss\": 10.935200108422173, \"crf_ner_loss\": 7.4790145556132, \"crf_term_loss\": 3.2752224736743503, \"cls_rel_loss\": 0.18096306009425056, \"f1_ner\": 0.43491767629698663, \"f1_term\": 0.33554987212276216, \"f1_rel\": 0.5099601593625498, \"epoch\": 13, \"method\": \"KPM\"}, {\"val_loss\": 13.95655904199085, \"val_crf_ner_loss\": 9.359742932076001, \"val_crf_term_loss\": 4.346859073116832, \"val_cls_rel_loss\": 0.24995723257969765, \"loss\": 13.672428343031141, \"crf_ner_loss\": 9.416304641299778, \"crf_term_loss\": 4.106093181504144, \"cls_rel_loss\": 0.15003018039796087, \"f1_ner\": 0.4515727187791965, \"f1_term\": 0.3736749116607773, \"f1_rel\": 0.5114813147230977, \"epoch\": 14, \"method\": \"KPM\"}, {\"val_loss\": 13.93107981577407, \"val_crf_ner_loss\": 9.336562083585418, \"val_crf_term_loss\": 4.343606795707759, \"val_cls_rel_loss\": 0.2509109221235679, \"loss\": 11.906590170330471, \"crf_ner_loss\": 8.182677851782906, \"crf_term_loss\": 3.573321567641364, \"cls_rel_loss\": 0.1505906908876366, \"f1_ner\": 0.47516198704103674, \"f1_term\": 0.40602836879432624, \"f1_rel\": 0.5275800711743772, \"epoch\": 15, \"method\": \"KPM\"}, {\"val_loss\": 13.928269602086422, \"val_crf_ner_loss\": 9.317142190724393, \"val_crf_term_loss\": 4.366590955831708, \"val_cls_rel_loss\": 0.2445363311001854, \"loss\": 12.97074524561564, \"crf_ner_loss\": 8.915461487240261, \"crf_term_loss\": 3.8990012539757624, \"cls_rel_loss\": 0.15628258718384636, \"f1_ner\": 0.4880147510755992, \"f1_term\": 0.40456621004566207, \"f1_rel\": 0.5264562027567808, \"epoch\": 16, \"method\": \"KPM\"}, {\"val_loss\": 13.899771871357938, \"val_crf_ner_loss\": 9.292585501705643, \"val_crf_term_loss\": 4.348387460639007, \"val_cls_rel_loss\": 0.2587989359876535, \"loss\": 13.465459373262194, \"crf_ner_loss\": 9.277238872316149, \"crf_term_loss\": 4.026697039604187, \"cls_rel_loss\": 0.16152334958314896, \"f1_ner\": 0.5033313143549364, \"f1_term\": 0.37316176470588236, \"f1_rel\": 0.5272895467160038, \"epoch\": 17, \"method\": \"KPM\"}, {\"val_loss\": 13.900357114137524, \"val_crf_ner_loss\": 9.276182639337804, \"val_crf_term_loss\": 4.360796211409743, \"val_cls_rel_loss\": 0.26337817289533405, \"loss\": 11.405402104059855, \"crf_ner_loss\": 7.84924070040385, \"crf_term_loss\": 3.416629751523336, \"cls_rel_loss\": 0.13953178748488426, \"f1_ner\": 0.5406713032960387, \"f1_term\": 0.4014665444546288, \"f1_rel\": 0.5261277355962483, \"epoch\": 18, \"method\": \"KPM\"}, {\"val_loss\": 13.896165350057784, \"val_crf_ner_loss\": 9.257388033135964, \"val_crf_term_loss\": 4.384082302559901, \"val_cls_rel_loss\": 0.2546948533858696, \"loss\": 11.414863692389595, \"crf_ner_loss\": 7.842244174745348, \"crf_term_loss\": 3.435810407002767, \"cls_rel_loss\": 0.13680922778116333, \"f1_ner\": 0.5773386847792529, \"f1_term\": 0.3829145728643216, \"f1_rel\": 0.5267175572519084, \"epoch\": 19, \"method\": \"KPM\"}, {\"val_loss\": 13.877817710820775, \"val_crf_ner_loss\": 9.246109692719731, \"val_crf_term_loss\": 4.375795293028337, \"val_cls_rel_loss\": 0.255912614564826, \"loss\": 13.860608206854927, \"crf_ner_loss\": 9.544105344348484, \"crf_term_loss\": 4.17823080221812, \"cls_rel_loss\": 0.13827224034402105, \"f1_ner\": 0.5676408556794216, \"f1_term\": 0.3892750744786495, \"f1_rel\": 0.5167381974248927, \"epoch\": 20, \"method\": \"KPM\"}, {\"val_loss\": 15.243806299501962, \"val_crf_ner_loss\": 10.107058149184624, \"val_crf_term_loss\": 4.683029660343253, \"val_cls_rel_loss\": 0.4537181549698767, \"loss\": 16.699586031248245, \"crf_ner_loss\": 11.24831004794553, \"crf_term_loss\": 4.924503230362487, \"cls_rel_loss\": 0.5267729591980255, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.0, \"epoch\": 1, \"method\": \"Manual\"}, {\"val_loss\": 14.912682700331194, \"val_crf_ner_loss\": 9.9304054705766, \"val_crf_term_loss\": 4.586394315218404, \"val_cls_rel_loss\": 0.39588289452295233, \"loss\": 14.990882839230325, \"crf_ner_loss\": 9.938884138203353, \"crf_term_loss\": 4.628164531515656, \"cls_rel_loss\": 0.4238341283455169, \"f1_ner\": 0.0, \"f1_term\": 0.0, \"f1_rel\": 0.1015625, \"epoch\": 2, \"method\": \"Manual\"}, {\"val_loss\": 14.627378710865104, \"val_crf_ner_loss\": 9.815573285095883, \"val_crf_term_loss\": 4.461484249490891, \"val_cls_rel_loss\": 0.3503211245049525, \"loss\": 14.700915823737494, \"crf_ner_loss\": 9.804691932184234, \"crf_term_loss\": 4.521886180630691, \"cls_rel_loss\": 0.374337677689765, \"f1_ner\": 0.0, \"f1_term\": 0.05017103762827822, \"f1_rel\": 0.25273010920436817, \"epoch\": 3, \"method\": \"Manual\"}, {\"val_loss\": 14.398991758805993, \"val_crf_ner_loss\": 9.709294179930303, \"val_crf_term_loss\": 4.380614814096994, \"val_cls_rel_loss\": 0.3090829792684012, \"loss\": 14.475410674115738, \"crf_ner_loss\": 9.713599438289945, \"crf_term_loss\": 4.423988352576606, \"cls_rel_loss\": 0.33782292098450145, \"f1_ner\": 0.0, \"f1_term\": 0.14696223316912974, \"f1_rel\": 0.3793103448275862, \"epoch\": 4, \"method\": \"Manual\"}, {\"val_loss\": 14.195851969892962, \"val_crf_ner_loss\": 9.582875979207728, \"val_crf_term_loss\": 4.323448491792609, \"val_cls_rel_loss\": 0.2895272591688337, \"loss\": 14.247947795785588, \"crf_ner_loss\": 9.613109622927878, \"crf_term_loss\": 4.350397923009859, \"cls_rel_loss\": 0.284440088615143, \"f1_ner\": 0.0, \"f1_term\": 0.26815240833932424, \"f1_rel\": 0.4473684210526315, \"epoch\": 5, \"method\": \"Manual\"}, {\"val_loss\": 14.019642192951954, \"val_crf_ner_loss\": 9.483940947664916, \"val_crf_term_loss\": 4.283573719706848, \"val_cls_rel_loss\": 0.2521274812030096, \"loss\": 14.042691244495858, \"crf_ner_loss\": 9.504135097531107, \"crf_term_loss\": 4.2868207581609274, \"cls_rel_loss\": 0.2517352447235327, \"f1_ner\": 0.0460122699386503, \"f1_term\": 0.3209130580731789, \"f1_rel\": 0.5104602510460251, \"epoch\": 6, \"method\": \"Manual\"}, {\"val_loss\": 13.945621532245275, \"val_crf_ner_loss\": 9.416973201027751, \"val_crf_term_loss\": 4.291519514835664, \"val_cls_rel_loss\": 0.23712905654071892, \"loss\": 13.894049123036776, \"crf_ner_loss\": 9.421672182975055, \"crf_term_loss\": 4.242246555767471, \"cls_rel_loss\": 0.23013035835122034, \"f1_ner\": 0.14721873693015475, \"f1_term\": 0.4274809160305343, \"f1_rel\": 0.5217391304347826, \"epoch\": 7, \"method\": \"Manual\"}, {\"val_loss\": 13.847039943193867, \"val_crf_ner_loss\": 9.365032241292244, \"val_crf_term_loss\": 4.256806112553951, \"val_cls_rel_loss\": 0.2252013226495172, \"loss\": 13.77873220375116, \"crf_ner_loss\": 9.358538778565771, \"crf_term_loss\": 4.205774499358033, \"cls_rel_loss\": 0.21441889816908527, \"f1_ner\": 0.2721679334057184, \"f1_term\": 0.4627766599597586, \"f1_rel\": 0.5461254612546126, \"epoch\": 8, \"method\": \"Manual\"}, {\"val_loss\": 13.772800000044551, \"val_crf_ner_loss\": 9.32188389771176, \"val_crf_term_loss\": 4.203255200908131, \"val_cls_rel_loss\": 0.24766094710705053, \"loss\": 13.690944232529016, \"crf_ner_loss\": 9.309969462936731, \"crf_term_loss\": 4.179576609632094, \"cls_rel_loss\": 0.20139821935043062, \"f1_ner\": 0.34415186772810774, \"f1_term\": 0.4714754098360656, \"f1_rel\": 0.5033647375504711, \"epoch\": 9, \"method\": \"Manual\"}, {\"val_loss\": 13.694452710395312, \"val_crf_ner_loss\": 9.28409958755883, \"val_crf_term_loss\": 4.189713124811214, \"val_cls_rel_loss\": 0.22063998192766288, \"loss\": 13.6023733564418, \"crf_ner_loss\": 9.266674076052878, \"crf_term_loss\": 4.143799555387428, \"cls_rel_loss\": 0.1918996330216634, \"f1_ner\": 0.3637484586929716, \"f1_term\": 0.4625719769673704, \"f1_rel\": 0.5594906486271388, \"epoch\": 10, \"method\": \"Manual\"}, {\"val_loss\": 13.650851120913986, \"val_crf_ner_loss\": 9.250646060400635, \"val_crf_term_loss\": 4.17821693246382, \"val_cls_rel_loss\": 0.22198789789728876, \"loss\": 13.528535136215979, \"crf_ner_loss\": 9.231000474888644, \"crf_term_loss\": 4.122180232041174, \"cls_rel_loss\": 0.1753543517786822, \"f1_ner\": 0.39359408684939945, \"f1_term\": 0.5359349593495935, \"f1_rel\": 0.5620703598867771, \"epoch\": 11, \"method\": \"Manual\"}, {\"val_loss\": 13.651739012585939, \"val_crf_ner_loss\": 9.219236165067576, \"val_crf_term_loss\": 4.200671663249496, \"val_cls_rel_loss\": 0.23183101219852476, \"loss\": 13.46437355425718, \"crf_ner_loss\": 9.193118143424714, \"crf_term_loss\": 4.103265326657741, \"cls_rel_loss\": 0.16799015675088486, \"f1_ner\": 0.4190246839253462, \"f1_term\": 0.5431290898274838, \"f1_rel\": 0.554696132596685, \"epoch\": 12, \"method\": \"Manual\"}, {\"val_loss\": 13.602622683030846, \"val_crf_ner_loss\": 9.189025001804325, \"val_crf_term_loss\": 4.1845643494251, \"val_cls_rel_loss\": 0.22903330939529587, \"loss\": 13.393721086515797, \"crf_ner_loss\": 9.160476073944311, \"crf_term_loss\": 4.07530150653647, \"cls_rel_loss\": 0.15794322977391936, \"f1_ner\": 0.4428442844284428, \"f1_term\": 0.5249169435215947, \"f1_rel\": 0.5605665300037271, \"epoch\": 13, \"method\": \"Manual\"}, {\"val_loss\": 13.545565184015427, \"val_crf_ner_loss\": 9.161734525304642, \"val_crf_term_loss\": 4.164743535710077, \"val_cls_rel_loss\": 0.21908710266116763, \"loss\": 13.347055352849068, \"crf_ner_loss\": 9.13083744735169, \"crf_term_loss\": 4.064087275978473, \"cls_rel_loss\": 0.15213072224915455, \"f1_ner\": 0.4908709967075726, \"f1_term\": 0.5755251750583528, \"f1_rel\": 0.5927209705372617, \"epoch\": 14, \"method\": \"Manual\"}, {\"val_loss\": 13.566906312956426, \"val_crf_ner_loss\": 9.141436623830865, \"val_crf_term_loss\": 4.179225100217945, \"val_cls_rel_loss\": 0.24624451755607216, \"loss\": 13.293653714571068, \"crf_ner_loss\": 9.101718058689036, \"crf_term_loss\": 4.0504371982684235, \"cls_rel_loss\": 0.14149870814608156, \"f1_ner\": 0.5365555215662282, \"f1_term\": 0.5604080331526937, \"f1_rel\": 0.5522174535050072, \"epoch\": 15, \"method\": \"Manual\"}, {\"val_loss\": 13.528757687032657, \"val_crf_ner_loss\": 9.11671873078729, \"val_crf_term_loss\": 4.181043214171472, \"val_cls_rel_loss\": 0.23099547298285214, \"loss\": 13.262428640461653, \"crf_ner_loss\": 9.074218688251303, \"crf_term_loss\": 4.034358688395658, \"cls_rel_loss\": 0.15385115746971514, \"f1_ner\": 0.5820622330689443, \"f1_term\": 0.5869017632241814, \"f1_rel\": 0.5668409260642271, \"epoch\": 16, \"method\": \"Manual\"}, {\"val_loss\": 13.499076745805949, \"val_crf_ner_loss\": 9.098606285387582, \"val_crf_term_loss\": 4.168222627500548, \"val_cls_rel_loss\": 0.23224780355056707, \"loss\": 13.205760331462614, \"crf_ner_loss\": 9.047224957308323, \"crf_term_loss\": 4.024408325016927, \"cls_rel_loss\": 0.1341272007218368, \"f1_ner\": 0.6072849709213346, \"f1_term\": 0.5895424836601307, \"f1_rel\": 0.6030061892130858, \"epoch\": 17, \"method\": \"Manual\"}, {\"val_loss\": 13.490994383818911, \"val_crf_ner_loss\": 9.081056586147225, \"val_crf_term_loss\": 4.172767678316492, \"val_cls_rel_loss\": 0.23717018896645872, \"loss\": 13.162245880785607, \"crf_ner_loss\": 9.026262454849354, \"crf_term_loss\": 4.005062815096738, \"cls_rel_loss\": 0.13092073208565336, \"f1_ner\": 0.61817065684469, \"f1_term\": 0.5970826580226904, \"f1_rel\": 0.6050000000000001, \"epoch\": 18, \"method\": \"Manual\"}, {\"val_loss\": 13.477757064095378, \"val_crf_ner_loss\": 9.067503215622727, \"val_crf_term_loss\": 4.169223362511962, \"val_cls_rel_loss\": 0.24103034129978096, \"loss\": 13.123809293019686, \"crf_ner_loss\": 9.004541664672413, \"crf_term_loss\": 4.000660311403892, \"cls_rel_loss\": 0.11860741154967452, \"f1_ner\": 0.6316436251920123, \"f1_term\": 0.6051834399192191, \"f1_rel\": 0.5977108944467996, \"epoch\": 19, \"method\": \"Manual\"}, {\"val_loss\": 13.485108493888465, \"val_crf_ner_loss\": 9.052033807239393, \"val_crf_term_loss\": 4.186074289962323, \"val_cls_rel_loss\": 0.247000602475048, \"loss\": 13.101578980041065, \"crf_ner_loss\": 8.987195536387052, \"crf_term_loss\": 4.005926411786525, \"cls_rel_loss\": 0.10845713746204651, \"f1_ner\": 0.6299357208448118, \"f1_term\": 0.6049889485317335, \"f1_rel\": 0.5819032761310452, \"epoch\": 20, \"method\": \"Manual\"}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.HConcatChart(...)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "te_results_df = results_df[results_df[\"method\"].isin([\"KPM\", \"KPM*\", \"Manual\"])]\n",
    "\n",
    "ner_f1 = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"f1_ner:Q\", title=\"NER F1\", scale=alt.Scale(domain=[0, 0.65])),\n",
    "    color=alt.Color(\"method:N\", title=\"TE Method\")\n",
    ")\n",
    "\n",
    "term_f1 = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"f1_term:Q\", title=\"Terms F1\", scale=alt.Scale(domain=[0, 0.65])),\n",
    "    color=alt.Color(\"method:N\", title=\"TE Method\")\n",
    ")\n",
    "\n",
    "rel_f1 = alt.Chart(te_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"f1_rel:Q\", title=\"Rel F1\", scale=alt.Scale(domain=[0, 0.65])),\n",
    "    color=alt.Color(\"method:N\", title=\"TE Method\")\n",
    ")\n",
    "\n",
    "all_charts = ner_f1 | term_f1 | rel_f1\n",
    "\n",
    "all_charts.configure_title(\n",
    "    fontSize=24,\n",
    ").configure_header(\n",
    "    titleFontSize=22,\n",
    "    labelFontSize=20\n",
    ").configure_axis(\n",
    "    titleFontSize=18,\n",
    "    labelFontSize=18\n",
    ").configure_legend(\n",
    "    titleFontSize=20,\n",
    "    labelFontSize=18\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1_ner</th>\n",
       "      <th>f1_term</th>\n",
       "      <th>f1_rel</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>KPM*</th>\n",
       "      <td>0.585797</td>\n",
       "      <td>0.480303</td>\n",
       "      <td>0.536034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Manual</th>\n",
       "      <td>0.631644</td>\n",
       "      <td>0.605183</td>\n",
       "      <td>0.605000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          f1_ner   f1_term    f1_rel\n",
       "method                              \n",
       "KPM*    0.585797  0.480303  0.536034\n",
       "Manual  0.631644  0.605183  0.605000"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df[results_df[\"method\"].isin([\"KPM*\", \"Manual\"])][[\"method\", \"f1_ner\", \"f1_term\", \"f1_rel\"]].groupby(\"method\").max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[[\"method\", \"val_loss\"]].reset_index().pivot_table(index=\"epoch\", columns=\"method\", values=\"val_loss\").plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[[\"method\", \"f1_term\"]].reset_index().pivot_table(index=\"epoch\", columns=\"method\", values=\"f1_term\").plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[results_df[\"method\"] == \"kpm\"][[\"loss\", \"val_loss\"]].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[results_df[\"method\"] == \"positionrank\"][[\"loss\", \"val_loss\"]].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kpm_results_df = results_df[results_df[\"method\"] == \"KPM*\"][[\"epoch\", \"crf_ner_loss\", \"val_crf_ner_loss\"]]\n",
    "kpm_results_df.columns = [\"epoch\", \"Train\", \"Test\"]\n",
    "kpm_results_df = kpm_results_df.melt(id_vars=[\"epoch\"], value_vars=[\"Train\", \"Test\"]).rename(\n",
    "    columns={\"variable\": \"Set\", \"value\": \"Loss\"}\n",
    ")\n",
    "kpm_results_df.columns = [\"epoch\", \"Set\", \"Loss\"]\n",
    "ner_chart = alt.Chart(kpm_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"Loss:Q\", title=\"NER Loss\", scale=alt.Scale(domain=[7, 11])),\n",
    "    color=\"Set:N\"\n",
    ")\n",
    "\n",
    "kpm_results_df = results_df[results_df[\"method\"] == \"KPM*\"][[\"epoch\", \"crf_term_loss\", \"val_crf_term_loss\"]]\n",
    "kpm_results_df.columns = [\"epoch\", \"Train\", \"Test\"]\n",
    "kpm_results_df = kpm_results_df.melt(id_vars=[\"epoch\"], value_vars=[\"Train\", \"Test\"]).rename(\n",
    "    columns={\"variable\": \"Set\", \"value\": \"Loss\"}\n",
    ")\n",
    "kpm_results_df.columns = [\"epoch\", \"Set\", \"Loss\"]\n",
    "term_chart = alt.Chart(kpm_results_df).mark_line().encode(\n",
    "    x=\"epoch:Q\",\n",
    "    y=alt.Y(\"Loss:Q\", title=\"Terms Loss\", scale=alt.Scale(domain=[3, 5])),\n",
    "    color=\"Set:N\"\n",
    ")\n",
    "\n",
    "all_charts = ner_chart | term_chart\n",
    "all_charts.configure_title(\n",
    "    fontSize=24,\n",
    ").configure_header(\n",
    "    titleFontSize=22,\n",
    "    labelFontSize=20\n",
    ").configure_axis(\n",
    "    titleFontSize=18,\n",
    "    labelFontSize=18\n",
    ").configure_legend(\n",
    "    titleFontSize=20,\n",
    "    labelFontSize=18\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}