examples/conll04/inspect.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline = [\n",
    "    './log.20190801-154135/',\n",
    "    './log.archive/log.20190727-191215/',\n",
    "    './log.20190801-024113/'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "from pprint import pprint\n",
    "from emr.utils import Namespace\n",
    "\n",
    "\n",
    "global tmp_emr\n",
    "def show_setup(log_path):\n",
    "    with open(os.path.join(log_path, 'config.pkl'), 'rb') as fin:\n",
    "        config = Namespace(pickle.load(fin))\n",
    "    ret_config = config.deepclone()\n",
    "\n",
    "    source = config.pop('Source')\n",
    "    display(Markdown('## Setup'))\n",
    "    pprint(config)\n",
    "    if 'emr' in source:\n",
    "        global tmp_emr\n",
    "        tmp_emr = source.emr\n",
    "        %pycat tmp_emr\n",
    "\n",
    "    return ret_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from glob import glob\n",
    "import re\n",
    "from collections import defaultdict\n",
    "import pandas as pd\n",
    "from IPython.display import display, HTML, Markdown\n",
    "from emr.conll_stat import Conll04Stat\n",
    "\n",
    "\n",
    "def show_result(log_path, config=None):\n",
    "    display(Markdown('## Result'))\n",
    "\n",
    "    metrics_path = sorted(glob(os.path.join(log_path, 'metrics_epoch_*.json')), key=lambda p: os.path.getmtime(p)).pop()\n",
    "    with open(metrics_path, 'r') as fin:\n",
    "        metrics = json.load(fin)\n",
    "        #pprint(metrics)\n",
    "\n",
    "    def pop_regex(container, regex):\n",
    "        if not isinstance(regex, re.Pattern):\n",
    "            regex = re.compile(regex, re.MULTILINE | re.DOTALL)\n",
    "        for key in list(container):\n",
    "            match = regex.fullmatch(key)\n",
    "            if match:\n",
    "                yield match, container.pop(key)\n",
    "\n",
    "    preserve_string = {'live_in': 'live-in',\n",
    "                       'located_in': 'located-in',\n",
    "                       'orgbase_on': 'orgbase-on',\n",
    "                       'work_for': 'work-for'}\n",
    "\n",
    "    def preserve(string):\n",
    "        for k, v in preserve_string.items():\n",
    "            string = string.replace(k, v)\n",
    "        return string\n",
    "\n",
    "    metrics_suffix = [\n",
    "        'F1',\n",
    "        'P',\n",
    "        'R',\n",
    "        'Accuracy',\n",
    "        'loss',\n",
    "    ]\n",
    "\n",
    "    global_names = [\n",
    "        'training_cpu_memory_MB',\n",
    "        'training_duration',\n",
    "        'training_epochs',\n",
    "        'training_gpu_0_memory_MB',\n",
    "        'training_gpu_1_memory_MB',\n",
    "        'training_start_epoch',\n",
    "        'best_epoch',\n",
    "        'epoch',\n",
    "        'peak_cpu_memory_MB',\n",
    "        'peak_gpu_0_memory_MB',\n",
    "        'peak_gpu_1_memory_MB',\n",
    "    ]\n",
    "\n",
    "    fields = [\n",
    "        ('best', {'best'}),\n",
    "        ('train', {'training', 'validation'}),\n",
    "        ('inference', {'i'}),\n",
    "    ]\n",
    "\n",
    "    entities = {'people':'Peop', 'organization':'Org', 'location':'Loc', 'other':'Other'}\n",
    "    relations = {'kill':'Kill', 'live-in':'Live_In', 'work-for':'Work_For', 'orgbase-on':'OrgBased_In', 'located-in':'Located_In'}\n",
    "    er = entities.copy()\n",
    "    er.update(relations)\n",
    "    def parse_fields(string_list):\n",
    "        field_list = ['',] * 4\n",
    "        for i, (field, field_set) in enumerate(fields):\n",
    "            if field:\n",
    "                for j, string in enumerate(list(string_list)):\n",
    "                    if string in field_set:\n",
    "                        field_list[i] = string\n",
    "                        string_list.pop(j)\n",
    "        field_list[-1] = string_list.pop().strip()\n",
    "        assert not string_list\n",
    "        return field_list\n",
    "    entity_table = pd.DataFrame()\n",
    "    best_epoch = metrics['best_epoch']\n",
    "\n",
    "    global_data = {}\n",
    "    for name in global_names:\n",
    "        for match, value in pop_regex(metrics, name):\n",
    "            global_data[match.string] = value\n",
    "    global_table = pd.DataFrame.from_dict(global_data, orient='index')\n",
    "    display(Markdown('### Global information'))\n",
    "    display(HTML(global_table.to_html(header=False,)))\n",
    "\n",
    "    if config:\n",
    "        test_path = os.path.join(config.Data.relative_path, config.Data.valid_path)\n",
    "        stat = Conll04Stat()\n",
    "        stat_result = dict(stat(test_path))\n",
    "        \n",
    "        #pprint(stat_result)\n",
    "    else:\n",
    "        stat_result = None\n",
    "\n",
    "    P = {}\n",
    "    TP = {}\n",
    "    FP = {}\n",
    "    TPi = {}\n",
    "    FPi = {}\n",
    "\n",
    "    for suffix in metrics_suffix:\n",
    "        display(Markdown('#### ' + suffix))\n",
    "        entity_dict = defaultdict(list)\n",
    "        relation_dict = defaultdict(list)\n",
    "        other_dict = defaultdict(list)\n",
    "        entity_index = []\n",
    "        relation_index = []\n",
    "\n",
    "        total_dict = {}\n",
    "        for match, value in pop_regex(metrics, r'(.*)(?<!-)-?{}'.format(suffix)):\n",
    "            string_list = preserve(str(match[1])).split('_')\n",
    "            field_list = parse_fields(string_list)\n",
    "            total_dict[tuple(field_list)] = value\n",
    "\n",
    "        for field_tuple, value in sorted(total_dict.items(), key=lambda k: k[0][-1]):\n",
    "            field_list = list(field_tuple)\n",
    "            index = field_list.pop()\n",
    "            if index in entities:\n",
    "                if index not in entity_index:\n",
    "                    entity_index.append(index)\n",
    "                entity_dict[tuple(field_list)].append(value)\n",
    "            elif index in relations:\n",
    "                if index not in relation_index:\n",
    "                    relation_index.append(index)\n",
    "                relation_dict[tuple(field_list)].append(value)\n",
    "            elif not index:\n",
    "                other_dict[tuple(field_list)] = value\n",
    "            else:\n",
    "                assert False, 'Not here, sir!'\n",
    "\n",
    "            if index in er and index not in P:\n",
    "                try:\n",
    "                    match, count = next(pop_regex(stat_result, r'^.*(?i){}$'.format(er[index])))\n",
    "                    P[index] = count\n",
    "                except StopIteration as e:\n",
    "                    print(index)\n",
    "                    print(entities[index])\n",
    "                    print(r'^.*(?i){}$'.format(entities[index]))\n",
    "                    print(stat_result)\n",
    "                    raise e\n",
    "\n",
    "            #print(field_tuple)\n",
    "            if field_tuple == ('', 'validation', '', index) and suffix.lower() == 'p':\n",
    "                TP[index] = value * P[index]\n",
    "            if field_tuple == ('', 'validation', '', index) and suffix.lower() == 'r':\n",
    "                FP[index] = (1. / value - 1) * TP[index]\n",
    "            if field_tuple == ('', 'validation', 'i', index) and suffix.lower() == 'p':\n",
    "                TPi[index] = value * P[index]\n",
    "            if field_tuple == ('', 'validation', 'i', index) and suffix.lower() == 'r':\n",
    "                FPi[index] = (1. / value - 1) * TPi[index]\n",
    "\n",
    "        entity_table = pd.DataFrame.from_dict(entity_dict)\n",
    "        entity_table = entity_table.append(entity_table.mean(numeric_only=True), ignore_index=True)\n",
    "        entity_table.index = entity_index + ['mean',]\n",
    "\n",
    "        relation_table = pd.DataFrame.from_dict(relation_dict)\n",
    "        relation_table = relation_table.append(relation_table.mean(numeric_only=True), ignore_index=True)\n",
    "        relation_table.index = relation_index + ['mean',]\n",
    "\n",
    "        other_tabel = pd.DataFrame.from_dict(other_dict, orient='index')\n",
    "        if entity_dict: display(HTML(entity_table.to_html()))\n",
    "        if relation_dict: display(HTML(relation_table.to_html()))\n",
    "        if other_dict: display(HTML(other_tabel.to_html(header=False,)))\n",
    "\n",
    "    #pprint(P)\n",
    "    #pprint(TP)\n",
    "    #pprint(FP)\n",
    "    P_entities = sum([P[key] for key in entities])\n",
    "    P_relations = sum([P[key] for key in relations])\n",
    "    print('micro (no inference)')\n",
    "    TP_entities = sum([TP[key] for key in entities])\n",
    "    FP_entities = sum([FP[key] for key in entities])\n",
    "    p = TP_entities / P_entities\n",
    "    r = TP_entities / (TP_entities + FP_entities)\n",
    "    print('F1 entities', 2 * p * r / (p + r))\n",
    "    print('P entities', p)\n",
    "    print('R entities', r)\n",
    "    TP_relations = sum([TP[key] for key in relations])\n",
    "    FP_relations = sum([FP[key] for key in relations])\n",
    "    p = TP_relations / P_relations\n",
    "    r = TP_relations / (TP_relations + FP_relations)\n",
    "    print('F1 relations', 2 * p * r / (p + r))\n",
    "    print('P relations', p)\n",
    "    print('R relations', r)\n",
    "\n",
    "    print('micro (inference)')\n",
    "    TPi_entities = sum([TPi[key] for key in entities])\n",
    "    FPi_entities = sum([FPi[key] for key in entities])\n",
    "    pi = TPi_entities / P_entities\n",
    "    ri = TPi_entities / (TPi_entities + FPi_entities)\n",
    "    print('F1 entities', 2 * pi * ri / (pi + ri))\n",
    "    print('P entities', pi)\n",
    "    print('R entities', ri)\n",
    "    TPi_relations = sum([TPi[key] for key in relations])\n",
    "    FPi_relations = sum([FPi[key] for key in relations])\n",
    "    pi = TPi_relations / P_relations\n",
    "    ri = TPi_relations / (TPi_relations + FPi_relations)\n",
    "    print('F1 relations', 2 * pi * ri / (pi + ri))\n",
    "    print('P relations', pi)\n",
    "    print('R relations', ri)\n",
    "\n",
    "    display(Markdown('### What\\'s left'))\n",
    "    left_tabel = pd.DataFrame.from_dict(metrics, orient='index')\n",
    "    display(HTML(left_tabel.to_html(header=False,)))\n",
    "    \n",
    "    return (P_entities, P_relations,\n",
    "            TP_entities, TP_relations,\n",
    "            FP_entities, FP_relations,\n",
    "            TPi_entities, TPi_relations,\n",
    "            FPi_entities, FPi_relations)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# log_path = './log.archive/log.20190801-024113/'\n",
    "# show_setup(log_path)\n",
    "# show_result(log_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'./log.best.leakyrelu/log.20190802-074627'\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Setup"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Data': {'relative_path': 'data/EntityMentionRelation',\n",
      "          'train_path': 'conll04.corp_3_train.corp',\n",
      "          'valid_path': 'conll04.corp_3_test.corp'},\n",
      " 'Model': {'activation': LeakyReLU(negative_slope=0.01),\n",
      "           'compact': Namespace('layers':[48]),\n",
      "           'distance_emb_size': 8,\n",
      "           'dropout': 0.3,\n",
      "           'embedding_dim': 8,\n",
      "           'graph': {'balance_factor': 1.5,\n",
      "                     'focal_gamma': 2,\n",
      "                     'inference_interval': 100,\n",
      "                     'inference_training_set': False,\n",
      "                     'label_smoothing': 0.01},\n",
      "           'max_distance': 64,\n",
      "           'ngram': 5,\n",
      "           'pretrained_dims': Namespace('word':50),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'relconv': Namespace('layers':[None, None, None],'kernel_size':7),\n",
      "           'relemb': Namespace('emb_size':256),\n",
      "           'rnn': Namespace('layers':2,'bidirectional':True)},\n",
      " 'Train': {'iterator': Namespace('batch_size':8),\n",
      "           'optimizer': Namespace('type':adam,'lr':0.0001,'weight_decay':1e-05),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'scheduler': Namespace('type':reduce_on_plateau,'patience':10),\n",
      "           'trainer': {'num_epochs': 100,\n",
      "                       'patience': None,\n",
      "                       'serialization_dir': 'log.20190802-074627'}}}\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Result"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Global information"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_cpu_memory_MB</th>\n",
       "      <td>3787.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_duration</th>\n",
       "      <td>4:22:44.615911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_epochs</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_0_memory_MB</th>\n",
       "      <td>22876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_1_memory_MB</th>\n",
       "      <td>22993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_start_epoch</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_epoch</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_cpu_memory_MB</th>\n",
       "      <td>3787.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_0_memory_MB</th>\n",
       "      <td>22876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_1_memory_MB</th>\n",
       "      <td>22994</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28889/28889 [00:00<00:00, 173071.73it/s]\n",
      "1103it [00:00, 108418.30it/s]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "#### F1"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.970274</td>\n",
       "      <td>0.869277</td>\n",
       "      <td>0.842161</td>\n",
       "      <td>0.876529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.962963</td>\n",
       "      <td>0.783525</td>\n",
       "      <td>0.765316</td>\n",
       "      <td>0.781853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.939586</td>\n",
       "      <td>0.663763</td>\n",
       "      <td>0.648045</td>\n",
       "      <td>0.673145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.986607</td>\n",
       "      <td>0.907749</td>\n",
       "      <td>0.898409</td>\n",
       "      <td>0.909315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.964858</td>\n",
       "      <td>0.806079</td>\n",
       "      <td>0.788483</td>\n",
       "      <td>0.810211</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.612766</td>\n",
       "      <td>0.534884</td>\n",
       "      <td>0.493506</td>\n",
       "      <td>0.523810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.600473</td>\n",
       "      <td>0.408511</td>\n",
       "      <td>0.335821</td>\n",
       "      <td>0.415584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.635514</td>\n",
       "      <td>0.453039</td>\n",
       "      <td>0.424528</td>\n",
       "      <td>0.462428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.596319</td>\n",
       "      <td>0.484018</td>\n",
       "      <td>0.467005</td>\n",
       "      <td>0.479263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.633929</td>\n",
       "      <td>0.460733</td>\n",
       "      <td>0.208955</td>\n",
       "      <td>0.430939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.615800</td>\n",
       "      <td>0.468237</td>\n",
       "      <td>0.385963</td>\n",
       "      <td>0.462405</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### P"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.954989</td>\n",
       "      <td>0.855592</td>\n",
       "      <td>0.798193</td>\n",
       "      <td>0.869757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.944767</td>\n",
       "      <td>0.770245</td>\n",
       "      <td>0.740876</td>\n",
       "      <td>0.774379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.916134</td>\n",
       "      <td>0.716165</td>\n",
       "      <td>0.637363</td>\n",
       "      <td>0.738372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.978804</td>\n",
       "      <td>0.901099</td>\n",
       "      <td>0.887545</td>\n",
       "      <td>0.905405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.948674</td>\n",
       "      <td>0.810775</td>\n",
       "      <td>0.765994</td>\n",
       "      <td>0.821978</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.418182</td>\n",
       "      <td>0.413043</td>\n",
       "      <td>0.415094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.585253</td>\n",
       "      <td>0.380952</td>\n",
       "      <td>0.283019</td>\n",
       "      <td>0.393443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.641509</td>\n",
       "      <td>0.414141</td>\n",
       "      <td>0.346154</td>\n",
       "      <td>0.439560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.531729</td>\n",
       "      <td>0.424000</td>\n",
       "      <td>0.446602</td>\n",
       "      <td>0.422764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.581967</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.358974</td>\n",
       "      <td>0.453488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.577183</td>\n",
       "      <td>0.419122</td>\n",
       "      <td>0.369558</td>\n",
       "      <td>0.424870</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### R"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.986057</td>\n",
       "      <td>0.883408</td>\n",
       "      <td>0.891256</td>\n",
       "      <td>0.883408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.981873</td>\n",
       "      <td>0.797271</td>\n",
       "      <td>0.791423</td>\n",
       "      <td>0.789474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.964271</td>\n",
       "      <td>0.618506</td>\n",
       "      <td>0.659091</td>\n",
       "      <td>0.618506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.994536</td>\n",
       "      <td>0.914498</td>\n",
       "      <td>0.909541</td>\n",
       "      <td>0.913259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.981684</td>\n",
       "      <td>0.803421</td>\n",
       "      <td>0.812828</td>\n",
       "      <td>0.801162</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.699029</td>\n",
       "      <td>0.741935</td>\n",
       "      <td>0.612903</td>\n",
       "      <td>0.709677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.616505</td>\n",
       "      <td>0.440367</td>\n",
       "      <td>0.412844</td>\n",
       "      <td>0.440367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.629630</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.548781</td>\n",
       "      <td>0.487805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.678771</td>\n",
       "      <td>0.563830</td>\n",
       "      <td>0.489362</td>\n",
       "      <td>0.553191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.696078</td>\n",
       "      <td>0.463158</td>\n",
       "      <td>0.147368</td>\n",
       "      <td>0.410526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.664003</td>\n",
       "      <td>0.541858</td>\n",
       "      <td>0.442252</td>\n",
       "      <td>0.520313</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### Accuracy"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.997948</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.991853</td>\n",
       "      <td>0.992369</td>\n",
       "      <td>0.989756</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.998685</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.992231</td>\n",
       "      <td>0.992231</td>\n",
       "      <td>0.991441</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.997413</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.986731</td>\n",
       "      <td>0.987281</td>\n",
       "      <td>0.984841</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.999263</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.994844</td>\n",
       "      <td>0.994947</td>\n",
       "      <td>0.994294</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.998327</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.991415</td>\n",
       "      <td>0.991707</td>\n",
       "      <td>0.990083</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.999955</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999923</td>\n",
       "      <td>0.999923</td>\n",
       "      <td>0.999925</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.999917</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999866</td>\n",
       "      <td>0.999870</td>\n",
       "      <td>0.999828</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.999942</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999904</td>\n",
       "      <td>0.999910</td>\n",
       "      <td>0.999882</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.999919</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999891</td>\n",
       "      <td>0.999891</td>\n",
       "      <td>0.999899</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.999939</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999901</td>\n",
       "      <td>0.999901</td>\n",
       "      <td>0.999898</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.999935</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999897</td>\n",
       "      <td>0.999899</td>\n",
       "      <td>0.999886</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### loss"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(, training, )</th>\n",
       "      <td>0.006487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(, validation, )</th>\n",
       "      <td>0.093760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(best, validation, )</th>\n",
       "      <td>0.056474</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "micro (no inference)\n",
      "F1 entities 0.8169534993907781\n",
      "P entities 0.8227256656409658\n",
      "R entities 0.8112617628096548\n",
      "F1 relations 0.45980419681068285\n",
      "P relations 0.418118428995167\n",
      "R relations 0.5107224431944692\n",
      "micro (inference)\n",
      "F1 entities 0.8211110749225532\n",
      "P entities 0.8340095426397243\n",
      "R entities 0.8086054963206492\n",
      "F1 relations 0.4537868275814641\n",
      "P relations 0.42417712064620056\n",
      "R relations 0.48784056468407694\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### What's left"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>validation_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_validation_\\n[ epoch ]</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'./log.best.leakyrelu/log.20190802-074517'\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Setup"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Data': {'relative_path': 'data/EntityMentionRelation',\n",
      "          'train_path': 'conll04.corp_2_train.corp',\n",
      "          'valid_path': 'conll04.corp_2_test.corp'},\n",
      " 'Model': {'activation': LeakyReLU(negative_slope=0.01),\n",
      "           'compact': Namespace('layers':[48]),\n",
      "           'distance_emb_size': 8,\n",
      "           'dropout': 0.3,\n",
      "           'embedding_dim': 8,\n",
      "           'graph': {'balance_factor': 1.5,\n",
      "                     'focal_gamma': 2,\n",
      "                     'inference_interval': 100,\n",
      "                     'inference_training_set': False,\n",
      "                     'label_smoothing': 0.01},\n",
      "           'max_distance': 64,\n",
      "           'ngram': 5,\n",
      "           'pretrained_dims': Namespace('word':50),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'relconv': Namespace('layers':[None, None, None],'kernel_size':7),\n",
      "           'relemb': Namespace('emb_size':256),\n",
      "           'rnn': Namespace('layers':2,'bidirectional':True)},\n",
      " 'Train': {'iterator': Namespace('batch_size':8),\n",
      "           'optimizer': Namespace('type':adam,'lr':0.0001,'weight_decay':1e-05),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'scheduler': Namespace('type':reduce_on_plateau,'patience':10),\n",
      "           'trainer': {'num_epochs': 100,\n",
      "                       'patience': None,\n",
      "                       'serialization_dir': 'log.20190802-074517'}}}\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Result"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Global information"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_cpu_memory_MB</th>\n",
       "      <td>3777.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_duration</th>\n",
       "      <td>4:21:30.098732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_epochs</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_0_memory_MB</th>\n",
       "      <td>22876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_1_memory_MB</th>\n",
       "      <td>22993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_start_epoch</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_epoch</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_cpu_memory_MB</th>\n",
       "      <td>3777.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_0_memory_MB</th>\n",
       "      <td>22876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_1_memory_MB</th>\n",
       "      <td>22994</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 29106/29106 [00:00<00:00, 171665.51it/s]\n",
      "1103it [00:00, 111528.59it/s]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "#### F1"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.973879</td>\n",
       "      <td>0.849620</td>\n",
       "      <td>0.847162</td>\n",
       "      <td>0.851646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.963844</td>\n",
       "      <td>0.767241</td>\n",
       "      <td>0.720703</td>\n",
       "      <td>0.768398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.947239</td>\n",
       "      <td>0.632911</td>\n",
       "      <td>0.625717</td>\n",
       "      <td>0.635616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.987713</td>\n",
       "      <td>0.919780</td>\n",
       "      <td>0.904481</td>\n",
       "      <td>0.921750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.968169</td>\n",
       "      <td>0.792388</td>\n",
       "      <td>0.774516</td>\n",
       "      <td>0.794353</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.651911</td>\n",
       "      <td>0.550336</td>\n",
       "      <td>0.390625</td>\n",
       "      <td>0.554054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.587134</td>\n",
       "      <td>0.453333</td>\n",
       "      <td>0.372093</td>\n",
       "      <td>0.450450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.683465</td>\n",
       "      <td>0.411429</td>\n",
       "      <td>0.322034</td>\n",
       "      <td>0.414201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.585859</td>\n",
       "      <td>0.442211</td>\n",
       "      <td>0.433735</td>\n",
       "      <td>0.437500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.641288</td>\n",
       "      <td>0.466019</td>\n",
       "      <td>0.117647</td>\n",
       "      <td>0.468750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.629931</td>\n",
       "      <td>0.464666</td>\n",
       "      <td>0.327227</td>\n",
       "      <td>0.464991</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### P"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.959293</td>\n",
       "      <td>0.839840</td>\n",
       "      <td>0.804608</td>\n",
       "      <td>0.841842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.944814</td>\n",
       "      <td>0.794643</td>\n",
       "      <td>0.678309</td>\n",
       "      <td>0.799550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.927885</td>\n",
       "      <td>0.693069</td>\n",
       "      <td>0.616129</td>\n",
       "      <td>0.704453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.981294</td>\n",
       "      <td>0.907005</td>\n",
       "      <td>0.860831</td>\n",
       "      <td>0.914425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.953322</td>\n",
       "      <td>0.808639</td>\n",
       "      <td>0.739969</td>\n",
       "      <td>0.815068</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.566434</td>\n",
       "      <td>0.445652</td>\n",
       "      <td>0.352113</td>\n",
       "      <td>0.450549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.562780</td>\n",
       "      <td>0.451327</td>\n",
       "      <td>0.388350</td>\n",
       "      <td>0.454545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.682390</td>\n",
       "      <td>0.418605</td>\n",
       "      <td>0.655172</td>\n",
       "      <td>0.437500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.527273</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.456522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.588710</td>\n",
       "      <td>0.413793</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.441176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.585517</td>\n",
       "      <td>0.434764</td>\n",
       "      <td>0.488218</td>\n",
       "      <td>0.448059</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### R"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.988915</td>\n",
       "      <td>0.859631</td>\n",
       "      <td>0.894467</td>\n",
       "      <td>0.861680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.983655</td>\n",
       "      <td>0.741667</td>\n",
       "      <td>0.768750</td>\n",
       "      <td>0.739583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.967419</td>\n",
       "      <td>0.582363</td>\n",
       "      <td>0.635607</td>\n",
       "      <td>0.579035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.994218</td>\n",
       "      <td>0.932919</td>\n",
       "      <td>0.952795</td>\n",
       "      <td>0.929193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.983552</td>\n",
       "      <td>0.779145</td>\n",
       "      <td>0.812905</td>\n",
       "      <td>0.777373</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.767772</td>\n",
       "      <td>0.719298</td>\n",
       "      <td>0.438596</td>\n",
       "      <td>0.719298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.613692</td>\n",
       "      <td>0.455357</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>0.446429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.684543</td>\n",
       "      <td>0.404494</td>\n",
       "      <td>0.213483</td>\n",
       "      <td>0.393258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.659091</td>\n",
       "      <td>0.440000</td>\n",
       "      <td>0.360000</td>\n",
       "      <td>0.420000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.704180</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.685856</td>\n",
       "      <td>0.510497</td>\n",
       "      <td>0.287178</td>\n",
       "      <td>0.495797</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### Accuracy"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.998236</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.989822</td>\n",
       "      <td>0.989959</td>\n",
       "      <td>0.989205</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.998692</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.992598</td>\n",
       "      <td>0.992666</td>\n",
       "      <td>0.990199</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.997736</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.986087</td>\n",
       "      <td>0.986327</td>\n",
       "      <td>0.984339</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.999324</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.995511</td>\n",
       "      <td>0.995648</td>\n",
       "      <td>0.994448</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.998497</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.991004</td>\n",
       "      <td>0.991150</td>\n",
       "      <td>0.989548</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.999957</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.999937</td>\n",
       "      <td>0.999926</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.999913</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999883</td>\n",
       "      <td>0.999884</td>\n",
       "      <td>0.999871</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.999950</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999902</td>\n",
       "      <td>0.999906</td>\n",
       "      <td>0.999924</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.999919</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999894</td>\n",
       "      <td>0.999897</td>\n",
       "      <td>0.999910</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.999940</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999895</td>\n",
       "      <td>0.999903</td>\n",
       "      <td>0.999914</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999902</td>\n",
       "      <td>0.999905</td>\n",
       "      <td>0.999909</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### loss"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(, training, )</th>\n",
       "      <td>0.005943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(, validation, )</th>\n",
       "      <td>0.101384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(best, validation, )</th>\n",
       "      <td>0.058375</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "micro (no inference)\n",
      "F1 entities 0.8057327255664135\n",
      "P entities 0.8203304554723011\n",
      "R entities 0.7916454436778214\n",
      "F1 relations 0.45468359510370054\n",
      "P relations 0.4350278721457081\n",
      "R relations 0.4761995652918304\n",
      "micro (inference)\n",
      "F1 entities 0.8074354878729711\n",
      "P entities 0.8263139031010021\n",
      "R entities 0.7894004181268928\n",
      "F1 relations 0.4540516286371761\n",
      "P relations 0.4484061805664429\n",
      "R relations 0.45984104197664344\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### What's left"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>validation_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_validation_\\n[ epoch ]</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'./log.best.leakyrelu/log.20190802-141118'\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Setup"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Data': {'relative_path': 'data/EntityMentionRelation',\n",
      "          'train_path': 'conll04.corp_4_train.corp',\n",
      "          'valid_path': 'conll04.corp_4_test.corp'},\n",
      " 'Model': {'activation': LeakyReLU(negative_slope=0.01),\n",
      "           'compact': Namespace('layers':[48]),\n",
      "           'distance_emb_size': 8,\n",
      "           'dropout': 0.3,\n",
      "           'embedding_dim': 8,\n",
      "           'graph': {'balance_factor': 1.5,\n",
      "                     'focal_gamma': 2,\n",
      "                     'inference_interval': 100,\n",
      "                     'inference_training_set': False,\n",
      "                     'label_smoothing': 0.01},\n",
      "           'max_distance': 64,\n",
      "           'ngram': 5,\n",
      "           'pretrained_dims': Namespace('word':50),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'relconv': Namespace('layers':[None, None, None],'kernel_size':7),\n",
      "           'relemb': Namespace('emb_size':256),\n",
      "           'rnn': Namespace('layers':2,'bidirectional':True)},\n",
      " 'Train': {'iterator': Namespace('batch_size':8),\n",
      "           'optimizer': Namespace('type':adam,'lr':0.0001,'weight_decay':1e-05),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'scheduler': Namespace('type':reduce_on_plateau,'patience':10),\n",
      "           'trainer': {'num_epochs': 100,\n",
      "                       'patience': None,\n",
      "                       'serialization_dir': 'log.20190802-141118'}}}\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Result"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Global information"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_cpu_memory_MB</th>\n",
       "      <td>3778.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_duration</th>\n",
       "      <td>4:23:35.508325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_epochs</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_0_memory_MB</th>\n",
       "      <td>22876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_1_memory_MB</th>\n",
       "      <td>23213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_start_epoch</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_epoch</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_cpu_memory_MB</th>\n",
       "      <td>3778.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_0_memory_MB</th>\n",
       "      <td>22876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_1_memory_MB</th>\n",
       "      <td>23214</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28134/28134 [00:00<00:00, 172260.21it/s]\n",
      "1103it [00:00, 109880.94it/s]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "#### F1"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.960360</td>\n",
       "      <td>0.860114</td>\n",
       "      <td>0.838940</td>\n",
       "      <td>0.862949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.951466</td>\n",
       "      <td>0.757185</td>\n",
       "      <td>0.722821</td>\n",
       "      <td>0.761239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.911795</td>\n",
       "      <td>0.681934</td>\n",
       "      <td>0.625434</td>\n",
       "      <td>0.687232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.976405</td>\n",
       "      <td>0.904762</td>\n",
       "      <td>0.890402</td>\n",
       "      <td>0.908862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.950007</td>\n",
       "      <td>0.800999</td>\n",
       "      <td>0.769399</td>\n",
       "      <td>0.805071</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.616016</td>\n",
       "      <td>0.585714</td>\n",
       "      <td>0.405405</td>\n",
       "      <td>0.594203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.582160</td>\n",
       "      <td>0.458874</td>\n",
       "      <td>0.405941</td>\n",
       "      <td>0.466368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.628478</td>\n",
       "      <td>0.548571</td>\n",
       "      <td>0.324786</td>\n",
       "      <td>0.535714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.615000</td>\n",
       "      <td>0.484848</td>\n",
       "      <td>0.358025</td>\n",
       "      <td>0.473684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.603989</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>0.439560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.609129</td>\n",
       "      <td>0.506511</td>\n",
       "      <td>0.319884</td>\n",
       "      <td>0.501906</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### P"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.942410</td>\n",
       "      <td>0.853457</td>\n",
       "      <td>0.816468</td>\n",
       "      <td>0.858031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.929259</td>\n",
       "      <td>0.778004</td>\n",
       "      <td>0.733598</td>\n",
       "      <td>0.788820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.884585</td>\n",
       "      <td>0.714032</td>\n",
       "      <td>0.546780</td>\n",
       "      <td>0.727768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.965550</td>\n",
       "      <td>0.902500</td>\n",
       "      <td>0.877900</td>\n",
       "      <td>0.909434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.930451</td>\n",
       "      <td>0.811998</td>\n",
       "      <td>0.743687</td>\n",
       "      <td>0.821013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.557621</td>\n",
       "      <td>0.455556</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.465909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.558559</td>\n",
       "      <td>0.449153</td>\n",
       "      <td>0.460674</td>\n",
       "      <td>0.472727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.655290</td>\n",
       "      <td>0.551724</td>\n",
       "      <td>0.655172</td>\n",
       "      <td>0.562500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.569444</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.371795</td>\n",
       "      <td>0.424528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.552083</td>\n",
       "      <td>0.391304</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.404040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.578599</td>\n",
       "      <td>0.453758</td>\n",
       "      <td>0.505862</td>\n",
       "      <td>0.465941</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### R"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.979008</td>\n",
       "      <td>0.866876</td>\n",
       "      <td>0.862683</td>\n",
       "      <td>0.867925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.974760</td>\n",
       "      <td>0.737452</td>\n",
       "      <td>0.712355</td>\n",
       "      <td>0.735521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.940731</td>\n",
       "      <td>0.652597</td>\n",
       "      <td>0.730519</td>\n",
       "      <td>0.650974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.987508</td>\n",
       "      <td>0.907035</td>\n",
       "      <td>0.903266</td>\n",
       "      <td>0.908291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.970502</td>\n",
       "      <td>0.790990</td>\n",
       "      <td>0.802206</td>\n",
       "      <td>0.790678</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.688073</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.820000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.607843</td>\n",
       "      <td>0.469027</td>\n",
       "      <td>0.362832</td>\n",
       "      <td>0.460177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.603774</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.215909</td>\n",
       "      <td>0.511364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.668478</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.345238</td>\n",
       "      <td>0.535714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.542169</td>\n",
       "      <td>0.060241</td>\n",
       "      <td>0.481928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.646967</td>\n",
       "      <td>0.589616</td>\n",
       "      <td>0.256844</td>\n",
       "      <td>0.561837</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### Accuracy"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.997316</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.990522</td>\n",
       "      <td>0.990734</td>\n",
       "      <td>0.988867</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.998283</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.991368</td>\n",
       "      <td>0.991579</td>\n",
       "      <td>0.990029</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.996226</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.986788</td>\n",
       "      <td>0.987140</td>\n",
       "      <td>0.981010</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.998701</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.994645</td>\n",
       "      <td>0.994891</td>\n",
       "      <td>0.993764</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.997632</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.990831</td>\n",
       "      <td>0.991086</td>\n",
       "      <td>0.988417</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.999955</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999941</td>\n",
       "      <td>0.999943</td>\n",
       "      <td>0.999955</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.999913</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999873</td>\n",
       "      <td>0.999879</td>\n",
       "      <td>0.999878</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.999945</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999920</td>\n",
       "      <td>0.999921</td>\n",
       "      <td>0.999920</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.999925</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999897</td>\n",
       "      <td>0.999899</td>\n",
       "      <td>0.999895</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.999932</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999891</td>\n",
       "      <td>0.999897</td>\n",
       "      <td>0.999914</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.999934</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999904</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.999912</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### loss"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(, training, )</th>\n",
       "      <td>0.008823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(, validation, )</th>\n",
       "      <td>0.083564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(best, validation, )</th>\n",
       "      <td>0.057998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "micro (no inference)\n",
      "F1 entities 0.8141019033084753\n",
      "P entities 0.8236608562704264\n",
      "R entities 0.8047622768212824\n",
      "F1 relations 0.49667982764326507\n",
      "P relations 0.4543789941015426\n",
      "R relations 0.5476652297430185\n",
      "micro (inference)\n",
      "F1 entities 0.8177401004366978\n",
      "P entities 0.8319641766700269\n",
      "R entities 0.8039942259368419\n",
      "F1 relations 0.49132223110464796\n",
      "P relations 0.46748652205798047\n",
      "R relations 0.5177191482415299\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### What's left"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>validation_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_validation_\\n[ epoch ]</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'./log.best.leakyrelu/log.20190802-141310'\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Setup"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Data': {'relative_path': 'data/EntityMentionRelation',\n",
      "          'train_path': 'conll04.corp_5_train.corp',\n",
      "          'valid_path': 'conll04.corp_5_test.corp'},\n",
      " 'Model': {'activation': LeakyReLU(negative_slope=0.01),\n",
      "           'compact': Namespace('layers':[48]),\n",
      "           'distance_emb_size': 8,\n",
      "           'dropout': 0.3,\n",
      "           'embedding_dim': 8,\n",
      "           'graph': {'balance_factor': 1.5,\n",
      "                     'focal_gamma': 2,\n",
      "                     'inference_interval': 100,\n",
      "                     'inference_training_set': False,\n",
      "                     'label_smoothing': 0.01},\n",
      "           'max_distance': 64,\n",
      "           'ngram': 5,\n",
      "           'pretrained_dims': Namespace('word':50),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'relconv': Namespace('layers':[None, None, None],'kernel_size':7),\n",
      "           'relemb': Namespace('emb_size':256),\n",
      "           'rnn': Namespace('layers':2,'bidirectional':True)},\n",
      " 'Train': {'iterator': Namespace('batch_size':8),\n",
      "           'optimizer': Namespace('type':adam,'lr':0.0001,'weight_decay':1e-05),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'scheduler': Namespace('type':reduce_on_plateau,'patience':10),\n",
      "           'trainer': {'num_epochs': 100,\n",
      "                       'patience': None,\n",
      "                       'serialization_dir': 'log.20190802-141310'}}}\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Result"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Global information"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_cpu_memory_MB</th>\n",
       "      <td>3780.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_duration</th>\n",
       "      <td>4:25:38.113460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_epochs</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_0_memory_MB</th>\n",
       "      <td>22876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_1_memory_MB</th>\n",
       "      <td>23214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_start_epoch</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_epoch</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_cpu_memory_MB</th>\n",
       "      <td>3780.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_0_memory_MB</th>\n",
       "      <td>22876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_1_memory_MB</th>\n",
       "      <td>23230</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 27626/27626 [00:00<00:00, 172552.10it/s]\n",
      "1103it [00:00, 109693.36it/s]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "#### F1"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.963078</td>\n",
       "      <td>0.859274</td>\n",
       "      <td>0.826438</td>\n",
       "      <td>0.860419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.957442</td>\n",
       "      <td>0.717480</td>\n",
       "      <td>0.695574</td>\n",
       "      <td>0.719101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.911545</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.651997</td>\n",
       "      <td>0.702034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.976967</td>\n",
       "      <td>0.918100</td>\n",
       "      <td>0.904277</td>\n",
       "      <td>0.922015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.952258</td>\n",
       "      <td>0.798713</td>\n",
       "      <td>0.769571</td>\n",
       "      <td>0.800892</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.544031</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.566210</td>\n",
       "      <td>0.430000</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.436548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.635155</td>\n",
       "      <td>0.506667</td>\n",
       "      <td>0.507463</td>\n",
       "      <td>0.510345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.607465</td>\n",
       "      <td>0.424870</td>\n",
       "      <td>0.355263</td>\n",
       "      <td>0.406417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.633663</td>\n",
       "      <td>0.389937</td>\n",
       "      <td>0.140625</td>\n",
       "      <td>0.405594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.597305</td>\n",
       "      <td>0.450295</td>\n",
       "      <td>0.358023</td>\n",
       "      <td>0.451781</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### P"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.944416</td>\n",
       "      <td>0.850394</td>\n",
       "      <td>0.784296</td>\n",
       "      <td>0.853610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.936477</td>\n",
       "      <td>0.747881</td>\n",
       "      <td>0.647059</td>\n",
       "      <td>0.753747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.886016</td>\n",
       "      <td>0.718919</td>\n",
       "      <td>0.623053</td>\n",
       "      <td>0.727106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.964002</td>\n",
       "      <td>0.918733</td>\n",
       "      <td>0.892761</td>\n",
       "      <td>0.925208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.932728</td>\n",
       "      <td>0.808982</td>\n",
       "      <td>0.736792</td>\n",
       "      <td>0.814918</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.487719</td>\n",
       "      <td>0.391892</td>\n",
       "      <td>0.350000</td>\n",
       "      <td>0.391892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.553571</td>\n",
       "      <td>0.401869</td>\n",
       "      <td>0.529412</td>\n",
       "      <td>0.413462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.493506</td>\n",
       "      <td>0.557377</td>\n",
       "      <td>0.513889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.560570</td>\n",
       "      <td>0.422680</td>\n",
       "      <td>0.482143</td>\n",
       "      <td>0.417582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.591029</td>\n",
       "      <td>0.360465</td>\n",
       "      <td>0.163636</td>\n",
       "      <td>0.414286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.563578</td>\n",
       "      <td>0.414083</td>\n",
       "      <td>0.416514</td>\n",
       "      <td>0.430222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### R"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.982493</td>\n",
       "      <td>0.868342</td>\n",
       "      <td>0.873367</td>\n",
       "      <td>0.867337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.979366</td>\n",
       "      <td>0.689453</td>\n",
       "      <td>0.751953</td>\n",
       "      <td>0.687500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.938589</td>\n",
       "      <td>0.682051</td>\n",
       "      <td>0.683761</td>\n",
       "      <td>0.678632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.990285</td>\n",
       "      <td>0.917469</td>\n",
       "      <td>0.916094</td>\n",
       "      <td>0.918845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.972683</td>\n",
       "      <td>0.789329</td>\n",
       "      <td>0.806294</td>\n",
       "      <td>0.788078</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.615044</td>\n",
       "      <td>0.690476</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.690476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.579439</td>\n",
       "      <td>0.462366</td>\n",
       "      <td>0.290323</td>\n",
       "      <td>0.462366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.645646</td>\n",
       "      <td>0.520548</td>\n",
       "      <td>0.465753</td>\n",
       "      <td>0.506849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.662921</td>\n",
       "      <td>0.427083</td>\n",
       "      <td>0.281250</td>\n",
       "      <td>0.395833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.682927</td>\n",
       "      <td>0.424658</td>\n",
       "      <td>0.123288</td>\n",
       "      <td>0.397260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.637195</td>\n",
       "      <td>0.505026</td>\n",
       "      <td>0.332123</td>\n",
       "      <td>0.490557</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### Accuracy"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.997538</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.989802</td>\n",
       "      <td>0.989910</td>\n",
       "      <td>0.986847</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.998501</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.989982</td>\n",
       "      <td>0.990090</td>\n",
       "      <td>0.987856</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.996195</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.987676</td>\n",
       "      <td>0.987856</td>\n",
       "      <td>0.984613</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.998709</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.995712</td>\n",
       "      <td>0.995928</td>\n",
       "      <td>0.994919</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.997736</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.990793</td>\n",
       "      <td>0.990946</td>\n",
       "      <td>0.988559</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.999943</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999942</td>\n",
       "      <td>0.999942</td>\n",
       "      <td>0.999940</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.999907</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999886</td>\n",
       "      <td>0.999889</td>\n",
       "      <td>0.999910</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.999940</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999926</td>\n",
       "      <td>0.999929</td>\n",
       "      <td>0.999934</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.999926</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999889</td>\n",
       "      <td>0.999889</td>\n",
       "      <td>0.999902</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.999937</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999903</td>\n",
       "      <td>0.999915</td>\n",
       "      <td>0.999890</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.999931</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999909</td>\n",
       "      <td>0.999913</td>\n",
       "      <td>0.999915</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### loss"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(, training, )</th>\n",
       "      <td>0.008755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(, validation, )</th>\n",
       "      <td>0.090606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(best, validation, )</th>\n",
       "      <td>0.063112</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "micro (no inference)\n",
      "F1 entities 0.8141090340815642\n",
      "P entities 0.8221153894313137\n",
      "R entities 0.8062571180689106\n",
      "F1 relations 0.4428901890517155\n",
      "P relations 0.41578391621220334\n",
      "R relations 0.4737772300407637\n",
      "micro (inference)\n",
      "F1 entities 0.8160464721434189\n",
      "P entities 0.8276850030370619\n",
      "R entities 0.8047307140444998\n",
      "F1 relations 0.44262156777154454\n",
      "P relations 0.43171363918787603\n",
      "R relations 0.4540949967892047\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### What's left"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>validation_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_validation_\\n[ epoch ]</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'./log.best.leakyrelu/log.20190801-154135'\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Setup"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Data': {'relative_path': 'data/EntityMentionRelation',\n",
      "          'train_path': 'conll04.corp_1_train.corp',\n",
      "          'valid_path': 'conll04.corp_1_test.corp'},\n",
      " 'Model': {'activation': LeakyReLU(negative_slope=0.01),\n",
      "           'compact': Namespace('layers':[48]),\n",
      "           'distance_emb_size': 8,\n",
      "           'dropout': 0.3,\n",
      "           'embedding_dim': 8,\n",
      "           'graph': {'balance_factor': 1.5,\n",
      "                     'focal_gamma': 2,\n",
      "                     'inference_interval': 100,\n",
      "                     'inference_training_set': False,\n",
      "                     'label_smoothing': 0.01},\n",
      "           'max_distance': 64,\n",
      "           'ngram': 5,\n",
      "           'pretrained_dims': Namespace('word':50),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'relconv': Namespace('layers':[None, None, None],'kernel_size':7),\n",
      "           'relemb': Namespace('emb_size':256),\n",
      "           'rnn': Namespace('layers':2,'bidirectional':True)},\n",
      " 'Train': {'iterator': Namespace('batch_size':8),\n",
      "           'optimizer': Namespace('lr':0.0001,'weight_decay':1e-05),\n",
      "           'pretrained_files': Namespace('word':data/glove.6B/glove.6B.50d.txt),\n",
      "           'scheduler': Namespace('patience':10),\n",
      "           'trainer': {'num_epochs': 100,\n",
      "                       'patience': None,\n",
      "                       'serialization_dir': 'log.20190801-154135'}}}\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "## Result"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "### Global information"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_cpu_memory_MB</th>\n",
       "      <td>3776.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_duration</th>\n",
       "      <td>4:23:22.944098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_epochs</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_0_memory_MB</th>\n",
       "      <td>22872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_gpu_1_memory_MB</th>\n",
       "      <td>22228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>training_start_epoch</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_epoch</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epoch</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_cpu_memory_MB</th>\n",
       "      <td>3776.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_0_memory_MB</th>\n",
       "      <td>22872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peak_gpu_1_memory_MB</th>\n",
       "      <td>22228</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28486/28486 [00:00<00:00, 172519.32it/s]\n",
      "1104it [00:00, 113554.16it/s]\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "#### F1"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.970683</td>\n",
       "      <td>0.871134</td>\n",
       "      <td>0.849952</td>\n",
       "      <td>0.877138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.961995</td>\n",
       "      <td>0.754274</td>\n",
       "      <td>0.676007</td>\n",
       "      <td>0.751880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.932503</td>\n",
       "      <td>0.640884</td>\n",
       "      <td>0.626427</td>\n",
       "      <td>0.644860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.985169</td>\n",
       "      <td>0.918054</td>\n",
       "      <td>0.913346</td>\n",
       "      <td>0.919717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.962587</td>\n",
       "      <td>0.796086</td>\n",
       "      <td>0.766433</td>\n",
       "      <td>0.798399</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.643326</td>\n",
       "      <td>0.549618</td>\n",
       "      <td>0.486486</td>\n",
       "      <td>0.553846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.609756</td>\n",
       "      <td>0.435644</td>\n",
       "      <td>0.260163</td>\n",
       "      <td>0.443299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.692308</td>\n",
       "      <td>0.476821</td>\n",
       "      <td>0.362205</td>\n",
       "      <td>0.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.640382</td>\n",
       "      <td>0.442105</td>\n",
       "      <td>0.330935</td>\n",
       "      <td>0.430939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.644628</td>\n",
       "      <td>0.496552</td>\n",
       "      <td>0.307692</td>\n",
       "      <td>0.476923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.646080</td>\n",
       "      <td>0.480148</td>\n",
       "      <td>0.349496</td>\n",
       "      <td>0.473309</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### P"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.957208</td>\n",
       "      <td>0.851815</td>\n",
       "      <td>0.785331</td>\n",
       "      <td>0.862385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.942600</td>\n",
       "      <td>0.767391</td>\n",
       "      <td>0.579580</td>\n",
       "      <td>0.769231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.906641</td>\n",
       "      <td>0.683694</td>\n",
       "      <td>0.591680</td>\n",
       "      <td>0.699797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.974711</td>\n",
       "      <td>0.920411</td>\n",
       "      <td>0.904762</td>\n",
       "      <td>0.925065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.945290</td>\n",
       "      <td>0.805828</td>\n",
       "      <td>0.715338</td>\n",
       "      <td>0.814119</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.597561</td>\n",
       "      <td>0.486486</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.493151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.578947</td>\n",
       "      <td>0.407407</td>\n",
       "      <td>0.551724</td>\n",
       "      <td>0.430000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.680233</td>\n",
       "      <td>0.467532</td>\n",
       "      <td>0.433962</td>\n",
       "      <td>0.478261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.578834</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.377049</td>\n",
       "      <td>0.378641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.607792</td>\n",
       "      <td>0.423529</td>\n",
       "      <td>0.315789</td>\n",
       "      <td>0.442857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.608673</td>\n",
       "      <td>0.431991</td>\n",
       "      <td>0.435705</td>\n",
       "      <td>0.444582</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### R"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.984543</td>\n",
       "      <td>0.891350</td>\n",
       "      <td>0.926160</td>\n",
       "      <td>0.892405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.982205</td>\n",
       "      <td>0.741597</td>\n",
       "      <td>0.810924</td>\n",
       "      <td>0.735294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.959884</td>\n",
       "      <td>0.603120</td>\n",
       "      <td>0.665511</td>\n",
       "      <td>0.597920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.995853</td>\n",
       "      <td>0.915709</td>\n",
       "      <td>0.922095</td>\n",
       "      <td>0.914432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.980621</td>\n",
       "      <td>0.787944</td>\n",
       "      <td>0.831173</td>\n",
       "      <td>0.785013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\"></th>\n",
       "      <th>best</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>training</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "      <th>validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.696682</td>\n",
       "      <td>0.631579</td>\n",
       "      <td>0.473684</td>\n",
       "      <td>0.631579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.644028</td>\n",
       "      <td>0.468085</td>\n",
       "      <td>0.170213</td>\n",
       "      <td>0.457447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.704819</td>\n",
       "      <td>0.486486</td>\n",
       "      <td>0.310811</td>\n",
       "      <td>0.445946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.716578</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.294872</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.686217</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.516667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.689665</td>\n",
       "      <td>0.544922</td>\n",
       "      <td>0.309916</td>\n",
       "      <td>0.510328</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### Accuracy"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>0.998016</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.991294</td>\n",
       "      <td>0.991747</td>\n",
       "      <td>0.989205</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organization</th>\n",
       "      <td>0.998628</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.991991</td>\n",
       "      <td>0.991956</td>\n",
       "      <td>0.987116</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.997063</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.986419</td>\n",
       "      <td>0.986767</td>\n",
       "      <td>0.984051</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.999178</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.995543</td>\n",
       "      <td>0.995647</td>\n",
       "      <td>0.995229</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.998221</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.991312</td>\n",
       "      <td>0.991529</td>\n",
       "      <td>0.988900</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\"></th>\n",
       "      <th colspan=\"2\" halign=\"left\">best</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">training</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "      <th colspan=\"2\" halign=\"left\">validation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "      <th></th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kill</th>\n",
       "      <td>0.999960</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999943</td>\n",
       "      <td>0.999944</td>\n",
       "      <td>0.999944</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>live-in</th>\n",
       "      <td>0.999914</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999889</td>\n",
       "      <td>0.999895</td>\n",
       "      <td>0.999911</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>located-in</th>\n",
       "      <td>0.999949</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999923</td>\n",
       "      <td>0.999925</td>\n",
       "      <td>0.999921</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orgbase-on</th>\n",
       "      <td>0.999926</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999897</td>\n",
       "      <td>0.999900</td>\n",
       "      <td>0.999909</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>work-for</th>\n",
       "      <td>0.999937</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999929</td>\n",
       "      <td>0.999934</td>\n",
       "      <td>0.999921</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.999937</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999916</td>\n",
       "      <td>0.999919</td>\n",
       "      <td>0.999922</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "#### loss"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(, training, )</th>\n",
       "      <td>0.007011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(, validation, )</th>\n",
       "      <td>0.097879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(best, validation, )</th>\n",
       "      <td>0.061502</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "micro (no inference)\n",
      "F1 entities 0.8137015113165328\n",
      "P entities 0.8218287556464302\n",
      "R entities 0.80573343711228\n",
      "F1 relations 0.47306163130194284\n",
      "P relations 0.4277829002414525\n",
      "R relations 0.5290600545991905\n",
      "micro (inference)\n",
      "F1 entities 0.8160992698450831\n",
      "P entities 0.8303892620807064\n",
      "R entities 0.8022927839275104\n",
      "F1 relations 0.4667356121159803\n",
      "P relations 0.4408438103093917\n",
      "R relations 0.49585856059521904\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "### What's left"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>training_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>validation_\\n[ epoch ]</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>best_validation_\\n[ epoch ]</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Totle micro\n",
      "no inference\n",
      "0.8128904021680999 0.8221348941246259 0.8038514977759641\n",
      "0.46522431318921237 0.43050145560118835 0.5060398541367247\n",
      "inference\n",
      "0.8156614970080476 0.8300713685526366 0.8017433933987831\n",
      "0.46161819005826055 0.4426581872830866 0.48227506914010126\n"
     ]
    }
   ],
   "source": [
    "master_path = './log.best.leakyrelu/'\n",
    "P_entities_list = []\n",
    "P_relations_list = []\n",
    "TP_entities_list = []\n",
    "TP_relations_list = []\n",
    "FP_entities_list = []\n",
    "FP_relations_list = []\n",
    "TPi_entities_list = []\n",
    "TPi_relations_list = []\n",
    "FPi_entities_list = []\n",
    "FPi_relations_list = []\n",
    "for log_path in glob(os.path.join(master_path, 'log.*')):\n",
    "    pprint(log_path)\n",
    "    if os.path.isdir(log_path):\n",
    "        config = show_setup(log_path)\n",
    "        (P_entities, P_relations,\n",
    "         TP_entities, TP_relations,\n",
    "         FP_entities, FP_relations,\n",
    "         TPi_entities, TPi_relations,\n",
    "         FPi_entities, FPi_relations) = show_result(log_path, config)\n",
    "        P_entities_list.append(P_entities)\n",
    "        P_relations_list.append(P_relations)\n",
    "        TP_entities_list.append(TP_entities)\n",
    "        TP_relations_list.append(TP_relations)\n",
    "        FP_entities_list.append(FP_entities)\n",
    "        FP_relations_list.append(FP_relations)\n",
    "        TPi_entities_list.append(TPi_entities)\n",
    "        TPi_relations_list.append(TPi_relations)\n",
    "        FPi_entities_list.append(FPi_entities)\n",
    "        FPi_relations_list.append(FPi_relations)\n",
    "\n",
    "def mm(P, TP, FP):\n",
    "    p = sum(P)\n",
    "    tp = sum(TP)\n",
    "    fp = sum(FP)\n",
    "    pr = tp / p\n",
    "    rc = tp / (tp + fp)\n",
    "    f1 = 2 * pr * rc / (pr + rc)\n",
    "    return f1, pr, rc\n",
    "\n",
    "print('Totle micro')\n",
    "print('no inference')\n",
    "f1, pr, rc = mm(P_entities_list, TP_entities_list, FP_entities_list)\n",
    "print(f1, pr, rc)\n",
    "f1, pr, rc = mm(P_relations_list, TP_relations_list, FP_relations_list)\n",
    "print(f1, pr, rc)\n",
    "print('inference')\n",
    "f1, pr, rc = mm(P_entities_list, TPi_entities_list, FPi_entities_list)\n",
    "print(f1, pr, rc)\n",
    "f1, pr, rc = mm(P_relations_list, TPi_relations_list, FPi_relations_list)\n",
    "print(f1, pr, rc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}