PrintFinalScores.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#BASELINES\" data-toc-modified-id=\"BASELINES-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>BASELINES</a></span></li><li><span><a href=\"#ORACLE\" data-toc-modified-id=\"ORACLE-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>ORACLE</a></span></li><li><span><a href=\"#DOC-MODEL\" data-toc-modified-id=\"DOC-MODEL-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>DOC MODEL</a></span></li><li><span><a href=\"#BERT-/-SUM\" data-toc-modified-id=\"BERT-/-SUM-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>BERT / SUM</a></span></li><li><span><a href=\"#Ensemble\" data-toc-modified-id=\"Ensemble-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>Ensemble</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's compare outputs from different methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_dir = \"../../BSDATA_V2/score_data\" # Change me to your score_data file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BASELINES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>rouge-1</th>\n",
       "      <th>rouge-2</th>\n",
       "      <th>rouge-l</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f</th>\n",
       "      <th>f</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>sumbasic</td>\n",
       "      <td>30.74</td>\n",
       "      <td>14.16</td>\n",
       "      <td>23.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>lsa</td>\n",
       "      <td>32.64</td>\n",
       "      <td>15.69</td>\n",
       "      <td>26.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>textrank</td>\n",
       "      <td>34.35</td>\n",
       "      <td>17.77</td>\n",
       "      <td>27.80</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         rouge-1 rouge-2 rouge-l\n",
       "               f       f       f\n",
       "sumbasic   30.74   14.16   23.92\n",
       "lsa        32.64   15.69   26.26\n",
       "textrank   34.35   17.77   27.80"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_data = {}\n",
    "\n",
    "scores = pickle.load(open('/Users/anastassiakornilova/BSDATA_V2/score_data/baseline_us_test_data_final.jsonl_2000.pkl', 'rb'))\n",
    "\n",
    "for method in ['sumbasic', 'lsa', 'textrank']:\n",
    "    basic = pd.DataFrame(scores).loc[method].to_dict()\n",
    "    index = []\n",
    "    value_data = {}\n",
    "    for k, ds in basic.items():\n",
    "        for v1, inner in ds.items():\n",
    "            for v2, score in inner.items():\n",
    "                value_data[(k, v1, v2)] = [score]\n",
    "\n",
    "\n",
    "    data = pd.DataFrame(value_data)\n",
    "    d2 = data.stack(0)\n",
    "    data = d2.droplevel(0)\n",
    "    final_data[(method, cutoff)] = data.describe().loc['mean']\n",
    "\n",
    "finals = pd.DataFrame(final_data).T\n",
    "(finals.reorder_levels([1,0]).loc['2000'] * 100)[[('rouge-1', 'f'), ('rouge-2', 'f'), ('rouge-l', 'f')]].round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>rouge-1</th>\n",
       "      <th>rouge-2</th>\n",
       "      <th>rouge-l</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f</th>\n",
       "      <th>f</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>sumbasic</td>\n",
       "      <td>35.47</td>\n",
       "      <td>16.18</td>\n",
       "      <td>29.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>lsa</td>\n",
       "      <td>35.06</td>\n",
       "      <td>16.34</td>\n",
       "      <td>29.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>textrank</td>\n",
       "      <td>35.81</td>\n",
       "      <td>18.10</td>\n",
       "      <td>29.97</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         rouge-1 rouge-2 rouge-l\n",
       "               f       f       f\n",
       "sumbasic   35.47   16.18   29.98\n",
       "lsa        35.06   16.34   29.93\n",
       "textrank   35.81   18.10   29.97"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_data = {}\n",
    "scores = pickle.load(open('/Users/anastassiakornilova/BSDATA_V2/score_data/baseline_ca_test_data_final.jsonl_2000.pkl', 'rb'))\n",
    "\n",
    "for method in ['sumbasic', 'lsa', 'textrank']:\n",
    "    basic = pd.DataFrame(scores).loc[method].to_dict()\n",
    "    index = []\n",
    "    value_data = {}\n",
    "    for k, ds in basic.items():\n",
    "        for v1, inner in ds.items():\n",
    "            for v2, score in inner.items():\n",
    "                value_data[(k, v1, v2)] = [score]\n",
    "\n",
    "\n",
    "    data = pd.DataFrame(value_data)\n",
    "    d2 = data.stack(0)\n",
    "    data = d2.droplevel(0)\n",
    "    final_data[(method, cutoff)] = data.describe().loc['mean']\n",
    "\n",
    "\n",
    "       \n",
    "finals = pd.DataFrame(final_data).T\n",
    "\n",
    "(finals.reorder_levels([1,0]).loc['2000'] * 100)[[('rouge-1', 'f'), ('rouge-2', 'f'), ('rouge-l', 'f')]].round(2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_data = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ORACLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from billsum.post_process import greedy_summarize, mmr_selection\n",
    "\n",
    "import os\n",
    "import pandas as pd \n",
    "import pickle \n",
    "from rouge import Rouge\n",
    "rouge = Rouge()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-1</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-2</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-l</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>0.451162</td>\n",
       "      <td>0.409217</td>\n",
       "      <td>0.581974</td>\n",
       "      <td>0.287442</td>\n",
       "      <td>0.258087</td>\n",
       "      <td>0.391674</td>\n",
       "      <td>0.383756</td>\n",
       "      <td>0.382623</td>\n",
       "      <td>0.545277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>0.151196</td>\n",
       "      <td>0.172050</td>\n",
       "      <td>0.179562</td>\n",
       "      <td>0.151056</td>\n",
       "      <td>0.159995</td>\n",
       "      <td>0.182031</td>\n",
       "      <td>0.166989</td>\n",
       "      <td>0.171348</td>\n",
       "      <td>0.183019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>0.053763</td>\n",
       "      <td>0.029412</td>\n",
       "      <td>0.033784</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021670</td>\n",
       "      <td>0.021622</td>\n",
       "      <td>0.027027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>0.331217</td>\n",
       "      <td>0.281442</td>\n",
       "      <td>0.460136</td>\n",
       "      <td>0.162757</td>\n",
       "      <td>0.131630</td>\n",
       "      <td>0.254732</td>\n",
       "      <td>0.251822</td>\n",
       "      <td>0.252959</td>\n",
       "      <td>0.418605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>0.448353</td>\n",
       "      <td>0.391304</td>\n",
       "      <td>0.615146</td>\n",
       "      <td>0.274065</td>\n",
       "      <td>0.226859</td>\n",
       "      <td>0.404716</td>\n",
       "      <td>0.369770</td>\n",
       "      <td>0.359426</td>\n",
       "      <td>0.576271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>0.572146</td>\n",
       "      <td>0.528382</td>\n",
       "      <td>0.720688</td>\n",
       "      <td>0.402361</td>\n",
       "      <td>0.357567</td>\n",
       "      <td>0.525641</td>\n",
       "      <td>0.516198</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.684211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>0.829971</td>\n",
       "      <td>0.922078</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.748515</td>\n",
       "      <td>0.842912</td>\n",
       "      <td>0.977376</td>\n",
       "      <td>0.824194</td>\n",
       "      <td>0.915584</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           rouge-1                                rouge-2               \\\n",
       "                 f            p            r            f            p   \n",
       "count  4506.000000  4506.000000  4506.000000  4506.000000  4506.000000   \n",
       "mean      0.451162     0.409217     0.581974     0.287442     0.258087   \n",
       "std       0.151196     0.172050     0.179562     0.151056     0.159995   \n",
       "min       0.053763     0.029412     0.033784     0.000000     0.000000   \n",
       "25%       0.331217     0.281442     0.460136     0.162757     0.131630   \n",
       "50%       0.448353     0.391304     0.615146     0.274065     0.226859   \n",
       "75%       0.572146     0.528382     0.720688     0.402361     0.357567   \n",
       "max       0.829971     0.922078     1.000000     0.748515     0.842912   \n",
       "\n",
       "                        rouge-l                            \n",
       "                 r            f            p            r  \n",
       "count  4506.000000  4506.000000  4506.000000  4506.000000  \n",
       "mean      0.391674     0.383756     0.382623     0.545277  \n",
       "std       0.182031     0.166989     0.171348     0.183019  \n",
       "min       0.000000     0.021670     0.021622     0.027027  \n",
       "25%       0.254732     0.251822     0.252959     0.418605  \n",
       "50%       0.404716     0.369770     0.359426     0.576271  \n",
       "75%       0.525641     0.516198     0.500000     0.684211  \n",
       "max       0.977376     0.824194     0.915584     1.000000  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prefix = '../../BSDATA_V2/'\n",
    "locality='us'\n",
    "# Load in the data\n",
    "test_data = pd.read_json(prefix + 'clean_final/{}_test_data_final.jsonl'.format(locality), lines=True)\n",
    "\n",
    "# CA data gets formatted weird - fix just in case\n",
    "if 'bill_id' not in test_data.columns:\n",
    "    test_data['bill_id'] = test_data['external_id']\n",
    "\n",
    "test_data.set_index('bill_id', inplace=True)\n",
    "test_sents = pickle.load(open(prefix + 'sent_data/{}_test_sent_scores.pkl'.format(locality), 'rb'))\n",
    "\n",
    "final_scores = {}\n",
    "for bill_id, sents in test_sents.items():\n",
    "\n",
    "    doc = [v[1] for v in sents]\n",
    "\n",
    "    scores = [v[2] for v in sents]\n",
    "    scores = [s['rouge-2']['p'] for s in scores]\n",
    "\n",
    "    sent_texts = [v[0] for v in sents]\n",
    "\n",
    "    summary = test_data.loc[bill_id]['clean_summary']\n",
    "\n",
    "    final_sum = ' '.join(mmr_selection(sent_texts, scores))\n",
    "    rs = rouge.get_scores([final_sum],[summary])[0]\n",
    "\n",
    "    final_scores[bill_id] = rs\n",
    "\n",
    "for k, ds in final_scores.items():\n",
    "    for v1, inner in ds.items():\n",
    "        for v2, score in inner.items():\n",
    "            value_data[(k, v1, v2)] = [score]\n",
    "\n",
    "\n",
    "data = pd.DataFrame(value_data)\n",
    "d2 = data.stack(0)\n",
    "data = d2.droplevel(0)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-1</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-2</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-l</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "      <td>4506.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>0.493797</td>\n",
       "      <td>0.449995</td>\n",
       "      <td>0.629631</td>\n",
       "      <td>0.333616</td>\n",
       "      <td>0.308500</td>\n",
       "      <td>0.438344</td>\n",
       "      <td>0.427797</td>\n",
       "      <td>0.427927</td>\n",
       "      <td>0.595821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>0.143365</td>\n",
       "      <td>0.191147</td>\n",
       "      <td>0.124857</td>\n",
       "      <td>0.140890</td>\n",
       "      <td>0.175149</td>\n",
       "      <td>0.139147</td>\n",
       "      <td>0.164603</td>\n",
       "      <td>0.188586</td>\n",
       "      <td>0.126805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>0.053763</td>\n",
       "      <td>0.029412</td>\n",
       "      <td>0.185185</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021670</td>\n",
       "      <td>0.021622</td>\n",
       "      <td>0.178899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>0.403465</td>\n",
       "      <td>0.302469</td>\n",
       "      <td>0.538568</td>\n",
       "      <td>0.231262</td>\n",
       "      <td>0.167933</td>\n",
       "      <td>0.336646</td>\n",
       "      <td>0.308569</td>\n",
       "      <td>0.282091</td>\n",
       "      <td>0.503186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>0.509859</td>\n",
       "      <td>0.450000</td>\n",
       "      <td>0.633333</td>\n",
       "      <td>0.337808</td>\n",
       "      <td>0.287560</td>\n",
       "      <td>0.433512</td>\n",
       "      <td>0.446708</td>\n",
       "      <td>0.423841</td>\n",
       "      <td>0.596154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.593700</td>\n",
       "      <td>0.723881</td>\n",
       "      <td>0.439305</td>\n",
       "      <td>0.431818</td>\n",
       "      <td>0.533013</td>\n",
       "      <td>0.555294</td>\n",
       "      <td>0.568345</td>\n",
       "      <td>0.688412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>0.829971</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.759878</td>\n",
       "      <td>0.985612</td>\n",
       "      <td>0.901235</td>\n",
       "      <td>0.824194</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.966667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           rouge-1                                rouge-2               \\\n",
       "                 f            p            r            f            p   \n",
       "count  4506.000000  4506.000000  4506.000000  4506.000000  4506.000000   \n",
       "mean      0.493797     0.449995     0.629631     0.333616     0.308500   \n",
       "std       0.143365     0.191147     0.124857     0.140890     0.175149   \n",
       "min       0.053763     0.029412     0.185185     0.000000     0.000000   \n",
       "25%       0.403465     0.302469     0.538568     0.231262     0.167933   \n",
       "50%       0.509859     0.450000     0.633333     0.337808     0.287560   \n",
       "75%       0.600000     0.593700     0.723881     0.439305     0.431818   \n",
       "max       0.829971     1.000000     0.966667     0.759878     0.985612   \n",
       "\n",
       "                        rouge-l                            \n",
       "                 r            f            p            r  \n",
       "count  4506.000000  4506.000000  4506.000000  4506.000000  \n",
       "mean      0.438344     0.427797     0.427927     0.595821  \n",
       "std       0.139147     0.164603     0.188586     0.126805  \n",
       "min       0.000000     0.021670     0.021622     0.178899  \n",
       "25%       0.336646     0.308569     0.282091     0.503186  \n",
       "50%       0.433512     0.446708     0.423841     0.596154  \n",
       "75%       0.533013     0.555294     0.568345     0.688412  \n",
       "max       0.901235     0.824194     1.000000     0.966667  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prefix = '../../BSDATA_V2/'\n",
    "locality='ca'\n",
    "# Load in the data\n",
    "test_data = pd.read_json(prefix + 'clean_final/{}_test_data_final.jsonl'.format(locality), lines=True)\n",
    "\n",
    "# CA data gets formatted weird - fix just in case\n",
    "if 'bill_id' not in test_data.columns:\n",
    "    test_data['bill_id'] = test_data['external_id']\n",
    "\n",
    "test_data.set_index('bill_id', inplace=True)\n",
    "test_sents = pickle.load(open(prefix + 'sent_data/{}_test_sent_scores.pkl'.format(locality), 'rb'))\n",
    "\n",
    "final_scores = {}\n",
    "for bill_id, sents in test_sents.items():\n",
    "\n",
    "    doc = [v[1] for v in sents]\n",
    "\n",
    "    scores = [v[2] for v in sents]\n",
    "    scores = [s['rouge-2']['p'] for s in scores]\n",
    "\n",
    "    sent_texts = [v[0] for v in sents]\n",
    "\n",
    "    summary = test_data.loc[bill_id]['clean_summary']\n",
    "\n",
    "    final_sum = ' '.join(mmr_selection(sent_texts, scores))\n",
    "    rs = rouge.get_scores([final_sum],[summary])[0]\n",
    "\n",
    "    final_scores[bill_id] = rs\n",
    "\n",
    "for k, ds in final_scores.items():\n",
    "    for v1, inner in ds.items():\n",
    "        for v2, score in inner.items():\n",
    "            value_data[(k, v1, v2)] = [score]\n",
    "\n",
    "\n",
    "data = pd.DataFrame(value_data)\n",
    "d2 = data.stack(0)\n",
    "data = d2.droplevel(0)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DOC MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-1</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-2</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-l</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>0.385140</td>\n",
       "      <td>0.326103</td>\n",
       "      <td>0.541571</td>\n",
       "      <td>0.213765</td>\n",
       "      <td>0.179310</td>\n",
       "      <td>0.323095</td>\n",
       "      <td>0.314836</td>\n",
       "      <td>0.300917</td>\n",
       "      <td>0.499184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>0.108212</td>\n",
       "      <td>0.128941</td>\n",
       "      <td>0.133100</td>\n",
       "      <td>0.098893</td>\n",
       "      <td>0.102997</td>\n",
       "      <td>0.136487</td>\n",
       "      <td>0.120424</td>\n",
       "      <td>0.123591</td>\n",
       "      <td>0.132710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>0.046784</td>\n",
       "      <td>0.025806</td>\n",
       "      <td>0.144000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.026053</td>\n",
       "      <td>0.025806</td>\n",
       "      <td>0.128000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>0.311111</td>\n",
       "      <td>0.226277</td>\n",
       "      <td>0.443182</td>\n",
       "      <td>0.140762</td>\n",
       "      <td>0.098940</td>\n",
       "      <td>0.223214</td>\n",
       "      <td>0.222234</td>\n",
       "      <td>0.205674</td>\n",
       "      <td>0.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>0.389776</td>\n",
       "      <td>0.320513</td>\n",
       "      <td>0.535088</td>\n",
       "      <td>0.204301</td>\n",
       "      <td>0.162712</td>\n",
       "      <td>0.305195</td>\n",
       "      <td>0.318446</td>\n",
       "      <td>0.293706</td>\n",
       "      <td>0.490909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>0.459574</td>\n",
       "      <td>0.415493</td>\n",
       "      <td>0.631206</td>\n",
       "      <td>0.278539</td>\n",
       "      <td>0.243542</td>\n",
       "      <td>0.402985</td>\n",
       "      <td>0.397708</td>\n",
       "      <td>0.384615</td>\n",
       "      <td>0.585106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>0.758621</td>\n",
       "      <td>0.817568</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.600962</td>\n",
       "      <td>0.661538</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.744649</td>\n",
       "      <td>0.810811</td>\n",
       "      <td>0.966667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           rouge-1                                rouge-2               \\\n",
       "                 f            p            r            f            p   \n",
       "count  3269.000000  3269.000000  3269.000000  3269.000000  3269.000000   \n",
       "mean      0.385140     0.326103     0.541571     0.213765     0.179310   \n",
       "std       0.108212     0.128941     0.133100     0.098893     0.102997   \n",
       "min       0.046784     0.025806     0.144000     0.000000     0.000000   \n",
       "25%       0.311111     0.226277     0.443182     0.140762     0.098940   \n",
       "50%       0.389776     0.320513     0.535088     0.204301     0.162712   \n",
       "75%       0.459574     0.415493     0.631206     0.278539     0.243542   \n",
       "max       0.758621     0.817568     0.966667     0.600962     0.661538   \n",
       "\n",
       "                        rouge-l                            \n",
       "                 r            f            p            r  \n",
       "count  3269.000000  3269.000000  3269.000000  3269.000000  \n",
       "mean      0.323095     0.314836     0.300917     0.499184  \n",
       "std       0.136487     0.120424     0.123591     0.132710  \n",
       "min       0.000000     0.026053     0.025806     0.128000  \n",
       "25%       0.223214     0.222234     0.205674     0.400000  \n",
       "50%       0.305195     0.318446     0.293706     0.490909  \n",
       "75%       0.402985     0.397708     0.384615     0.585106  \n",
       "max       0.888889     0.744649     0.810811     0.966667  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = pd.read_pickle(score_dir + '/us_test_feature_model_res.pkl')\n",
    "index = []\n",
    "value_data = {}\n",
    "for k, ds in scores.items():\n",
    "    for v1, inner in ds.items():\n",
    "        for v2, score in inner.items():\n",
    "            value_data[(k, v1, v2)] = [score]\n",
    "\n",
    "\n",
    "data = pd.DataFrame(value_data)\n",
    "d2 = data.stack(0)\n",
    "data = d2.droplevel(0)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-1</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-2</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-l</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>0.383451</td>\n",
       "      <td>0.383131</td>\n",
       "      <td>0.414978</td>\n",
       "      <td>0.197641</td>\n",
       "      <td>0.204926</td>\n",
       "      <td>0.213342</td>\n",
       "      <td>0.328002</td>\n",
       "      <td>0.346724</td>\n",
       "      <td>0.374255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>0.086871</td>\n",
       "      <td>0.126518</td>\n",
       "      <td>0.097332</td>\n",
       "      <td>0.085951</td>\n",
       "      <td>0.113529</td>\n",
       "      <td>0.092188</td>\n",
       "      <td>0.091935</td>\n",
       "      <td>0.123359</td>\n",
       "      <td>0.094861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>0.074419</td>\n",
       "      <td>0.043243</td>\n",
       "      <td>0.190789</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.033144</td>\n",
       "      <td>0.032432</td>\n",
       "      <td>0.164474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>0.329502</td>\n",
       "      <td>0.299145</td>\n",
       "      <td>0.346369</td>\n",
       "      <td>0.135084</td>\n",
       "      <td>0.121457</td>\n",
       "      <td>0.148718</td>\n",
       "      <td>0.266734</td>\n",
       "      <td>0.260870</td>\n",
       "      <td>0.307692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>0.381743</td>\n",
       "      <td>0.376543</td>\n",
       "      <td>0.402010</td>\n",
       "      <td>0.188976</td>\n",
       "      <td>0.192453</td>\n",
       "      <td>0.200369</td>\n",
       "      <td>0.328292</td>\n",
       "      <td>0.338983</td>\n",
       "      <td>0.361582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>0.438356</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.475000</td>\n",
       "      <td>0.252918</td>\n",
       "      <td>0.270386</td>\n",
       "      <td>0.265403</td>\n",
       "      <td>0.387962</td>\n",
       "      <td>0.423841</td>\n",
       "      <td>0.433498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>0.639098</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.757576</td>\n",
       "      <td>0.541033</td>\n",
       "      <td>0.987552</td>\n",
       "      <td>0.638710</td>\n",
       "      <td>0.598439</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.737374</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           rouge-1                                rouge-2               \\\n",
       "                 f            p            r            f            p   \n",
       "count  1237.000000  1237.000000  1237.000000  1237.000000  1237.000000   \n",
       "mean      0.383451     0.383131     0.414978     0.197641     0.204926   \n",
       "std       0.086871     0.126518     0.097332     0.085951     0.113529   \n",
       "min       0.074419     0.043243     0.190789     0.000000     0.000000   \n",
       "25%       0.329502     0.299145     0.346369     0.135084     0.121457   \n",
       "50%       0.381743     0.376543     0.402010     0.188976     0.192453   \n",
       "75%       0.438356     0.461538     0.475000     0.252918     0.270386   \n",
       "max       0.639098     1.000000     0.757576     0.541033     0.987552   \n",
       "\n",
       "                        rouge-l                            \n",
       "                 r            f            p            r  \n",
       "count  1237.000000  1237.000000  1237.000000  1237.000000  \n",
       "mean      0.213342     0.328002     0.346724     0.374255  \n",
       "std       0.092188     0.091935     0.123359     0.094861  \n",
       "min       0.000000     0.033144     0.032432     0.164474  \n",
       "25%       0.148718     0.266734     0.260870     0.307692  \n",
       "50%       0.200369     0.328292     0.338983     0.361582  \n",
       "75%       0.265403     0.387962     0.423841     0.433498  \n",
       "max       0.638710     0.598439     1.000000     0.737374  "
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = pd.read_pickle(score_dir + '/ca_test_feature_model_res.pkl')\n",
    "index = []\n",
    "value_data = {}\n",
    "for k, ds in scores.items():\n",
    "    for v1, inner in ds.items():\n",
    "        for v2, score in inner.items():\n",
    "            value_data[(k, v1, v2)] = [score]\n",
    "\n",
    "\n",
    "data = pd.DataFrame(value_data)\n",
    "d2 = data.stack(0)\n",
    "data = d2.droplevel(0)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT / SUM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = pd.read_pickle(score_dir + '/us_bert_scores.pkl')\n",
    "index = []\n",
    "value_data = {}\n",
    "for k, ds in scores.items():\n",
    "    for v1, inner in ds.items():\n",
    "        for v2, score in inner.items():\n",
    "            value_data[(k, v1, v2)] = [score]\n",
    "\n",
    "\n",
    "data = pd.DataFrame(value_data)\n",
    "d2 = data.stack(0)\n",
    "data = d2.droplevel(0)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-1</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-2</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-l</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "      <td>3269.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>0.408045</td>\n",
       "      <td>0.347718</td>\n",
       "      <td>0.566831</td>\n",
       "      <td>0.238299</td>\n",
       "      <td>0.201133</td>\n",
       "      <td>0.355569</td>\n",
       "      <td>0.337340</td>\n",
       "      <td>0.323110</td>\n",
       "      <td>0.526168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>0.112874</td>\n",
       "      <td>0.135614</td>\n",
       "      <td>0.134506</td>\n",
       "      <td>0.106368</td>\n",
       "      <td>0.112183</td>\n",
       "      <td>0.142350</td>\n",
       "      <td>0.127314</td>\n",
       "      <td>0.131061</td>\n",
       "      <td>0.135368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>0.057803</td>\n",
       "      <td>0.031847</td>\n",
       "      <td>0.178899</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021694</td>\n",
       "      <td>0.021622</td>\n",
       "      <td>0.174312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>0.331606</td>\n",
       "      <td>0.244898</td>\n",
       "      <td>0.469880</td>\n",
       "      <td>0.160991</td>\n",
       "      <td>0.114155</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.242265</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>0.415094</td>\n",
       "      <td>0.342857</td>\n",
       "      <td>0.560284</td>\n",
       "      <td>0.232099</td>\n",
       "      <td>0.184615</td>\n",
       "      <td>0.337500</td>\n",
       "      <td>0.341555</td>\n",
       "      <td>0.316901</td>\n",
       "      <td>0.516949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>0.486486</td>\n",
       "      <td>0.445205</td>\n",
       "      <td>0.658537</td>\n",
       "      <td>0.306533</td>\n",
       "      <td>0.274262</td>\n",
       "      <td>0.443350</td>\n",
       "      <td>0.429996</td>\n",
       "      <td>0.414966</td>\n",
       "      <td>0.618182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>0.782946</td>\n",
       "      <td>0.801370</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.730159</td>\n",
       "      <td>0.662791</td>\n",
       "      <td>0.901235</td>\n",
       "      <td>0.747498</td>\n",
       "      <td>0.773973</td>\n",
       "      <td>0.966667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           rouge-1                                rouge-2               \\\n",
       "                 f            p            r            f            p   \n",
       "count  3269.000000  3269.000000  3269.000000  3269.000000  3269.000000   \n",
       "mean      0.408045     0.347718     0.566831     0.238299     0.201133   \n",
       "std       0.112874     0.135614     0.134506     0.106368     0.112183   \n",
       "min       0.057803     0.031847     0.178899     0.000000     0.000000   \n",
       "25%       0.331606     0.244898     0.469880     0.160991     0.114155   \n",
       "50%       0.415094     0.342857     0.560284     0.232099     0.184615   \n",
       "75%       0.486486     0.445205     0.658537     0.306533     0.274262   \n",
       "max       0.782946     0.801370     0.966667     0.730159     0.662791   \n",
       "\n",
       "                        rouge-l                            \n",
       "                 r            f            p            r  \n",
       "count  3269.000000  3269.000000  3269.000000  3269.000000  \n",
       "mean      0.355569     0.337340     0.323110     0.526168  \n",
       "std       0.142350     0.127314     0.131061     0.135368  \n",
       "min       0.000000     0.021694     0.021622     0.174312  \n",
       "25%       0.250000     0.242265     0.222222     0.428571  \n",
       "50%       0.337500     0.341555     0.316901     0.516949  \n",
       "75%       0.443350     0.429996     0.414966     0.618182  \n",
       "max       0.901235     0.747498     0.773973     0.966667  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = pd.read_pickle('/Users/anastassiakornilova/BSDATA_V2/score_data/us_ensemble_scores.pkl')\n",
    "value_data = {}\n",
    "for k, ds in scores.items():\n",
    "    for v1, inner in ds.items():\n",
    "        for v2, score in inner.items():\n",
    "            value_data[(k, v1, v2)] = [score]\n",
    "\n",
    "\n",
    "data = pd.DataFrame(value_data)\n",
    "d2 = data.stack(0)\n",
    "data = d2.droplevel(0)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-1</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-2</th>\n",
       "      <th colspan=\"3\" halign=\"left\">rouge-l</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "      <td>1237.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>0.396520</td>\n",
       "      <td>0.398523</td>\n",
       "      <td>0.425103</td>\n",
       "      <td>0.214076</td>\n",
       "      <td>0.223440</td>\n",
       "      <td>0.228227</td>\n",
       "      <td>0.340509</td>\n",
       "      <td>0.361673</td>\n",
       "      <td>0.384143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>0.090687</td>\n",
       "      <td>0.131409</td>\n",
       "      <td>0.097673</td>\n",
       "      <td>0.092400</td>\n",
       "      <td>0.121173</td>\n",
       "      <td>0.098779</td>\n",
       "      <td>0.096799</td>\n",
       "      <td>0.129505</td>\n",
       "      <td>0.096584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>0.082353</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>0.164474</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.048599</td>\n",
       "      <td>0.047619</td>\n",
       "      <td>0.151316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>0.340278</td>\n",
       "      <td>0.309353</td>\n",
       "      <td>0.351916</td>\n",
       "      <td>0.147874</td>\n",
       "      <td>0.132296</td>\n",
       "      <td>0.155807</td>\n",
       "      <td>0.280733</td>\n",
       "      <td>0.273256</td>\n",
       "      <td>0.314815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>0.398773</td>\n",
       "      <td>0.393939</td>\n",
       "      <td>0.412844</td>\n",
       "      <td>0.207059</td>\n",
       "      <td>0.207469</td>\n",
       "      <td>0.213033</td>\n",
       "      <td>0.341457</td>\n",
       "      <td>0.354430</td>\n",
       "      <td>0.369863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>0.458221</td>\n",
       "      <td>0.482993</td>\n",
       "      <td>0.489796</td>\n",
       "      <td>0.275986</td>\n",
       "      <td>0.297170</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.402757</td>\n",
       "      <td>0.448052</td>\n",
       "      <td>0.445783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>0.657061</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.761194</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.984314</td>\n",
       "      <td>0.712644</td>\n",
       "      <td>0.642457</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.737374</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           rouge-1                                rouge-2               \\\n",
       "                 f            p            r            f            p   \n",
       "count  1237.000000  1237.000000  1237.000000  1237.000000  1237.000000   \n",
       "mean      0.396520     0.398523     0.425103     0.214076     0.223440   \n",
       "std       0.090687     0.131409     0.097673     0.092400     0.121173   \n",
       "min       0.082353     0.047619     0.164474     0.000000     0.000000   \n",
       "25%       0.340278     0.309353     0.351916     0.147874     0.132296   \n",
       "50%       0.398773     0.393939     0.412844     0.207059     0.207469   \n",
       "75%       0.458221     0.482993     0.489796     0.275986     0.297170   \n",
       "max       0.657061     1.000000     0.761194     0.520000     0.984314   \n",
       "\n",
       "                        rouge-l                            \n",
       "                 r            f            p            r  \n",
       "count  1237.000000  1237.000000  1237.000000  1237.000000  \n",
       "mean      0.228227     0.340509     0.361673     0.384143  \n",
       "std       0.098779     0.096799     0.129505     0.096584  \n",
       "min       0.000000     0.048599     0.047619     0.151316  \n",
       "25%       0.155807     0.280733     0.273256     0.314815  \n",
       "50%       0.213033     0.341457     0.354430     0.369863  \n",
       "75%       0.285714     0.402757     0.448052     0.445783  \n",
       "max       0.712644     0.642457     1.000000     0.737374  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = pd.read_pickle('/Users/anastassiakornilova/BSDATA_V2/score_data/ca_ensemble_scores.pkl')\n",
    "value_data = {}\n",
    "for k, ds in scores.items():\n",
    "    for v1, inner in ds.items():\n",
    "        for v2, score in inner.items():\n",
    "            value_data[(k, v1, v2)] = [score]\n",
    "\n",
    "\n",
    "data = pd.DataFrame(value_data)\n",
    "d2 = data.stack(0)\n",
    "data = d2.droplevel(0)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}