Evaluate_Statute_Triage_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f96c62d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: train\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Evaluate statute triage against gold standard\n",
    "'''\n",
    "\n",
    "import json\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import traceback\n",
    "import re\n",
    "import pandas as pd\n",
    "import requests\n",
    "\n",
    "TRAIN_TEST = \"train\"\n",
    "IS_USE_LIVE_BOT = False\n",
    "\n",
    "\n",
    "print(f\"Dataset: {TRAIN_TEST}\")\n",
    "\n",
    "df_gold_standard = pd.read_csv(f\"{TRAIN_TEST}_statute_gold_standard.csv\", encoding=\"utf-8\", sep=\"\\t\")\n",
    "\n",
    "df = pd.read_csv(f\"output_{TRAIN_TEST}_insolvency_bot_with_gpt-4.csv\", encoding=\"utf-8\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "295d821e",
   "metadata": {},
   "outputs": [],
   "source": [
    "if IS_USE_LIVE_BOT:\n",
    "    sys.path.append(\"../insolvency/\")\n",
    "    from insolvency_bot import get_matching_sources, construct_prompt, embeddings_lookup\n",
    "\n",
    "    bot_statutes = [\"\"] * len(df)\n",
    "\n",
    "    for idx in range(len(df)):\n",
    "        question = df.question_text.iloc[idx]\n",
    "        matching_sources = get_matching_sources(question)\n",
    "        dialogue_history = construct_prompt(question, matching_sources)\n",
    "        bot_statutes[idx] = \"|\".join(matching_sources[\"human_readable_s\"])\n",
    "    df[\"bot_statutes\"] = bot_statutes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "80a5d737",
   "metadata": {},
   "outputs": [],
   "source": [
    "re_num = re.compile(r'\\b[A-Z]?\\d+[A-Z]?\\b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9ce21482",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalise_statute_name(c):\n",
    "    c = re.sub(r'\\(.+', '', c)\n",
    "    statute = \"\"\n",
    "    if \"insolvency\" in c.lower() or \"IA\" in c:\n",
    "        statute = \"IA\"\n",
    "    if \"companies\" in c.lower() or \"CA\" in c:\n",
    "        statute = \"CA\"\n",
    "    matches = re_num.findall(c)\n",
    "    if len(matches) > 0:\n",
    "        for m in matches:\n",
    "            if m != \"2006\" and m != \"1986\":\n",
    "                return m + statute\n",
    "    return c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ee127d57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'124IA'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalise_statute_name('Insolvency Act 1986 section 124 (Winding Up of Companies Registered under the Companies Acts)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e5fa5b74",
   "metadata": {},
   "outputs": [],
   "source": [
    "question_to_key_statutes = dict(df_gold_standard.set_index(\"question_no\")[\"statutes\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bc34014d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.question_no.isin(question_to_key_statutes)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "225adf9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ground truth:\t\t ['154CA', '250CA']\n",
      "\tPredicted:\t ['123IA', '124IA', '154CA', '234IA', '371IA']\n",
      "Ground truth:\t\t ['124', '166', '6IA']\n",
      "\tPredicted:\t ['123IA', '124IA', '154CA', '222IA', '43CA']\n",
      "Ground truth:\t\t ['122IA', '123IA', '125IA', '143IA']\n",
      "\tPredicted:\t ['122IA', '123IA', '124IA', '95IA', 'A16IA']\n",
      "Ground truth:\t\t ['171CA', '250IA']\n",
      "\tPredicted:\t ['1187CA', '123IA', '124IA', '15', '373IA']\n",
      "Ground truth:\t\t ['122IA']\n",
      "\tPredicted:\t ['122IA', '123IA', '124IA', '251CA', 'A16IA']\n",
      "Ground truth:\t\t ['122IA', '123IA', '125IA']\n",
      "\tPredicted:\t ['122IA', '123IA', '124IA', '3CA', '58CA']\n",
      "Ground truth:\t\t ['171CA', '212IA', '238IA', '244IA', '423IA']\n",
      "\tPredicted:\t ['122IA', '123IA', '124IA', '154CA', '155CA']\n",
      "Ground truth:\t\t ['171CA', '212IA', '213IA', '214IA', '238IA', '244IA', '423IA']\n",
      "\tPredicted:\t ['123IA', '124IA', '393CA', '414CA', '472CA']\n",
      "Ground truth:\t\t ['122IA', '123IA', '171CA']\n",
      "\tPredicted:\t ['122IA', '123IA', '124IA', '251GIA', 'A18IA']\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "ctr = Counter()\n",
    "\n",
    "total_statutes_gt = [0] * len(df)\n",
    "total_statutes_retrieved = [0] * len(df)\n",
    "total_statutes_correct = [0] * len(df)\n",
    "\n",
    "for idx in range(len(df)):\n",
    "    q_no = df.question_no.iloc[idx]\n",
    "    \n",
    "    y_gt = question_to_key_statutes[q_no].split(\"|\")\n",
    "    \n",
    "    y_pred = df.bot_statutes.iloc[idx].split(\"|\")\n",
    "    \n",
    "    y_pred_normalised = set([normalise_statute_name(c) for c in y_pred])\n",
    "    y_gt_normalised = set([normalise_statute_name(c) for c in y_gt])\n",
    "    \n",
    "    for n in y_gt_normalised:\n",
    "        ctr[n] += 1\n",
    "    \n",
    "    print (\"Ground truth:\\t\\t\", sorted(y_gt_normalised))\n",
    "    print (\"\\tPredicted:\\t\", sorted(y_pred_normalised))\n",
    "    \n",
    "    total_statutes_gt[idx] = len(y_gt_normalised)\n",
    "    total_statutes_correct[idx] = len(y_pred_normalised.intersection(y_gt_normalised))\n",
    "    total_statutes_retrieved[idx] = len(y_pred_normalised)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "55a6f6f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"statutes_ground_truth\"] = total_statutes_gt\n",
    "df[\"statutes_retrieved\"] = total_statutes_retrieved\n",
    "df[\"statutes_correct\"] = total_statutes_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a9b9e7e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question_no</th>\n",
       "      <th>statutes_ground_truth</th>\n",
       "      <th>statutes_retrieved</th>\n",
       "      <th>statutes_correct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Q2</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Q3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Q4</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Q5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Q7</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Q9</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Q10</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Q12</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   question_no  statutes_ground_truth  statutes_retrieved  statutes_correct\n",
       "0           Q1                      2                   5                 1\n",
       "1           Q2                      3                   5                 0\n",
       "2           Q3                      4                   5                 2\n",
       "3           Q4                      2                   5                 0\n",
       "4           Q5                      1                   5                 1\n",
       "6           Q7                      3                   5                 2\n",
       "8           Q9                      5                   5                 0\n",
       "9          Q10                      7                   5                 0\n",
       "11         Q12                      3                   5                 2"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[[\"question_no\",\"statutes_ground_truth\", \"statutes_retrieved\", \"statutes_correct\" ]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3283d65e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 18%\n",
      "Recall: 27%\n"
     ]
    }
   ],
   "source": [
    "print (f\"Precision: {df.statutes_correct.sum() / df.statutes_retrieved.sum():.0%}\")\n",
    "print (f\"Recall: {df.statutes_correct.sum() / df.statutes_ground_truth.sum():.0%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33547e1c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}