i_get_caseidx.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "06f5c036-528b-4bf2-872b-c982f8342cfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re, os, glob, csv, math, requests, time, sys, random, json, datetime, urllib#, dryscrape\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup as bs\n",
    "from selenium import webdriver\n",
    "from sklearn.utils import shuffle      \n",
    "import numpy as np\n",
    "from nordvpn_switcher import initialize_VPN,rotate_VPN,terminate_VPN\n",
    "from selenium import webdriver \n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "# https://github.com/canlii/API_documentation/blob/master/EN.md\n",
    "\n",
    "\"\"\" custom functions:\"\"\"\n",
    "from webscraping_kit import write_htmlfile, write_json_tofile, write_driverhtmlfile\n",
    "from webscraping_kit import read_jsoncsv, read_htmlfile, read_htmlresponse\n",
    "from webscraping_kit import build_queries, query_canlii_api, save_canlii_response, parse_caseid, parse_canlii_responses, format_canlii_caseurls\n",
    "from webscraping_kit import parse_tabledata, get_tablerescount, get_tabledata\n",
    "from webscraping_kit import rmnl, get_apikeys, get_origin\n",
    "\"\"\" \n",
    "------------------------------------------------------------------------------------------------------------------\n",
    "webscraping functions: parse_tabledata(soups,casefolder), get_tablerescount(soup), get_tabledata(tablelink,tablefile)\n",
    "------------------------------------------------------------------------------------------------------------------\n",
    "canlii api functions: build_queries(canlii_key,start,end), query_canlii_api(queries), save_canlii_response(canliiapiresponse,file), \n",
    "                      parse_caseid(bscanliiapiresponse), parse_canlii_responses(mainpath), format_canlii_caseurls(row)\n",
    "------------------------------------------------------------------------------------------------------------------\n",
    "other functions: get_origin(row,tablecids)\n",
    "\"\"\"     \n",
    "\n",
    "mainpath = str(os.getcwd())+'/'\n",
    "tablefolder = mainpath + 'data/tables/'\n",
    "casefolder = mainpath + 'data/cases/'\n",
    "canlii_key = get_apikeys(mainpath + 'apikeys.txt','canlii_key')\n",
    "headers = get_apikeys(mainpath + 'apikeys.txt','headers')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60829975-4aa4-4846-a88b-4a2eac7a5391",
   "metadata": {},
   "source": [
    "<h2> 1.  create an index of cases to collect (2 ways) </h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6d1ca3f-1c23-444a-a6c6-fc2613226f17",
   "metadata": {},
   "source": [
    "<h3> a) webscraping CanLII </h3>\n",
    "- create the index by webscraping canlii for cases from the ONLTB by year <br>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "340d8a92-0571-4bfe-aa09-32f1cff8beba",
   "metadata": {},
   "source": [
    "<h4> i. get total count of cases for tribunal and note date </h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "80625d8e-bdf8-4864-b632-7328591038d4",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ONLTB', 'Landlord and Tenant Board', '2009 -', '2022-01-06', '36,437']\n",
      "2022-01-08 09:52:38.914691\n"
     ]
    }
   ],
   "source": [
    "\"\"\" get total count of canlii cases \"\"\"\n",
    "\"\"\" - link to table of legislation, courts, boards and tribunals, containing case count information \"\"\"\n",
    "canliilaw_casecountfile = mainpath + 'data/1_canliicasecount.html'\n",
    "canliilawlink = \"https://www.canlii.org/en/on/\"\n",
    "\n",
    "\"\"\" save the table to a file \"\"\"\n",
    "# canliilawresponse = requests.get(canliilawlink,headers=headers)\n",
    "# write_htmlfile(canliilawresponse,canliilaw_casecountfile)\n",
    "# print(canliilawresponse)\n",
    "# canliilawsoup = bs(canliilawresponse.text,'html.parser')\n",
    "canliilawsoup = read_htmlfile(canliilaw_casecountfile)\n",
    "\n",
    "canliionltb_casecount = [j for j in [i.strip() for i in [x for x in canliilawsoup.find_all('div',{'class':'row row-stripped py-1 ml-0 tribunalRow'}) if 'Landlord and Tenant Board' in x.text][0].text.split(\"\\n\")] if j]\n",
    "print(canliionltb_casecount) #['ONLTB', 'Landlord and Tenant Board', '2009 -', '2022-01-06', '36,437']\n",
    "print(datetime.datetime.now())\n",
    "\n",
    "# ['ONLTB', 'Landlord and Tenant Board', '2009 -', '2022-01-06', '36,437']\n",
    "# 2022-01-07 15:49:47.217678"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76686517-4375-4f4c-a36c-2a10f7a310dd",
   "metadata": {},
   "source": [
    "<h4> ii. save yearly tables of links to each case </h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c8b3d61e-ec18-4817-a00d-e79e03ad4fac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n"
     ]
    }
   ],
   "source": [
    "tablelinkprefix = \"https://www.canlii.org/en/on/onltb/nav/date/\"\n",
    "tablelinkyears =[tablelinkprefix + str(x) + '/' for x in list(range(2006,2022))]\n",
    "tdf = pd.DataFrame(tablelinkyears,columns=['tablelink'])\n",
    "tdf['year'] = tdf['tablelink'].apply(lambda x: int(x.split('/')[-2]))\n",
    "tdf['tablefile'] = tdf['year'].apply(lambda x: tablefolder + str(x) + '.html')\n",
    "tabledict = dict(zip(tdf['tablelink'],tdf['tablefile']))\n",
    "print(len(tabledict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04e5f035-078d-4246-81a3-b71b5e348ac5",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "settings = initialize_VPN(save=0,area_input=['complete rotation'],skip_settings=1)\n",
    "\n",
    "# rotate_VPN(settings,google_check=1)\n",
    "rotate_VPN(settings)\n",
    "\n",
    "for j,(k,v) in enumerate(tabledict.items()):\n",
    "    get_tabledata(k,v)\n",
    "    print('sleeping for 10 seconds...')\n",
    "    time.sleep(10)\n",
    "    print(j)\n",
    "    print('resuming...')\n",
    "\n",
    "# was missing 2021\n",
    "# rotate_VPN(settings)\n",
    "\n",
    "# get_tabledata('https://www.canlii.org/en/on/onltb/nav/date/2021/','/home/bunds/bundslab/ltb_env/ltb/data/tables/2021.html')\n",
    "terminate_VPN(settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3206f1b1-59b0-4a2d-9f41-1c9c0164ac06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There should be 36437 cases.\n",
      "The sum of all result counts: 36435\n",
      "There are 36435 cases.\n"
     ]
    }
   ],
   "source": [
    "tablefiles = glob.glob(tablefolder + '*.html')\n",
    "\n",
    "soups = []\n",
    "for i,f in enumerate(tablefiles):\n",
    "    soups.append([f,read_htmlfile(f)])\n",
    "\n",
    "tydf = parse_tabledata(soups,casefolder)\n",
    "tydf['table_result_count'] = tydf['table_result_count'].astype(int)\n",
    "\n",
    "print('There should be '+ str(re.sub(',','',canliionltb_casecount[-1]).strip()) + ' cases.')\n",
    "print('The sum of all result counts: '+ str(tydf[['tablefile','table_result_count']].drop_duplicates()['table_result_count'].sum()))\n",
    "print('There are '+ str(len(list(set(tydf['cid'].tolist())))) + ' cases.')\n",
    "\n",
    "tydf['tableurl'] = tydf['tablefile'].map(dict(zip(tdf['tablefile'],tdf['tablelink'])))\n",
    "tydf = tydf[['cid','fileno','year','tablefile','tableurl','table_result_count','caseoutfile','caseurl']]\n",
    "tydf.to_csv(mainpath + 'output/1_tablecaseidx.csv',sep='\\t',index=False,quoting=csv.QUOTE_ALL)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5446639a-9abd-42fe-8bf6-e73dc1705577",
   "metadata": {},
   "source": [
    "<h3> b) using the CanLII API </h3>\n",
    "- create the index and list of individual case urls from canlii <br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1bdbaf3d-befd-4ec5-80f2-7ccdebc18139",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\n",
      "https://api.canlii.org/v1/caseBrowse/en/onltb/?offset=0&resultCount=10000&decisionDateBefore=2022-01-01&decisionDateAfter=2020-12-31&api_key=CANLII_API_KEY"
     ]
    }
   ],
   "source": [
    "queries = build_queries(canlii_key,2006,2021)\n",
    "print(len(queries))\n",
    "print(queries[-1])\n",
    "\n",
    "apioutfile = mainpath + 'data/1_apicaseidx.csv'\n",
    "# query_canlii_api(queries,apioutfile,headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4346dd15-559f-4ba9-b393-233bc98d3c94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(33205, 6)\n"
     ]
    }
   ],
   "source": [
    "tapidf = parse_canlii_responses(apioutfile,casefolder)\n",
    "print(tapidf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "16721755-3dd7-4c20-aa92-fcbb06e53561",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36435\n",
      "33205\n",
      "There are 3295 cases collected by webscraping, not available through the canlii api.\n",
      "There are 65 cases collected by api, not available through webscraping.\n"
     ]
    }
   ],
   "source": [
    "tablecids = list(set(tydf['cid'].tolist()))\n",
    "print(len(tablecids))\n",
    "apicids = list(set(tapidf['cid'].tolist()))\n",
    "print(len(apicids))\n",
    "print(\"There are \"+str(len(list(set(tablecids)-set(apicids))))+\" cases collected by webscraping, not available through the canlii api.\")\n",
    "print(\"There are \"+str(len(list(set(apicids)-set(tablecids))))+\" cases collected by api, not available through webscraping.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "75e7ffba-4cf7-4576-8ef3-1bde4e056237",
   "metadata": {},
   "outputs": [],
   "source": [
    "tapidf.to_csv(mainpath + 'output/1_apicaseidx.csv',sep='\\t',index=False,quoting=csv.QUOTE_ALL)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "306ab3a2-d117-4735-b3a7-9cafe1a93b24",
   "metadata": {},
   "source": [
    "<h2> 2. merge results from both collections </h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1421b95c-b07f-4abf-a1c4-cd38263117d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(36499, 4)\n",
      "(33204, 4)\n",
      "(33204, 4)\n",
      "(36434, 4)\n",
      "(36434, 4)\n"
     ]
    }
   ],
   "source": [
    "tydf['caseurl'] = tydf['caseurl'].apply(lambda x: \"https://www.canlii.org/en/on/onltb/doc/\" + x.split('/doc/',1)[-1])\n",
    "tydf = tydf[['cid','year','caseoutfile','caseurl']]\n",
    "tydf = tydf[~tydf['caseurl'].str.contains('/doc/2005/2005canlii')]\n",
    "tapidf = tapidf[['cid','year','caseoutfile','caseurl']]\n",
    "tapidf = tapidf[tapidf['cid']!='2011canlii92502']\n",
    "\n",
    "df = pd.concat([tydf,tapidf]).drop_duplicates()#,on=['cid','caseurl','year','caseoutfile'],how='outer')\n",
    "print(df.shape)\n",
    "\n",
    "print(tapidf.shape)\n",
    "print(df[df['cid'].isin(tapidf['cid'].tolist())].shape)\n",
    "\n",
    "print(tydf.shape)\n",
    "print(df[df['cid'].isin(tydf['cid'].tolist())].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8aaf59f4-2c4c-490a-aba9-913d4cac8d57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(36499, 5)\n",
      "both         33139\n",
      "webscrape     3295\n",
      "api             65\n",
      "Name: source, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "tablecids = list(set(tydf['cid'].tolist()))\n",
    "apicids = list(set(tapidf['cid'].tolist()))\n",
    "tablematches = [x for x in tablecids if x in apicids]\n",
    "\n",
    "df['source'] = np.where(df['cid'].isin(tablematches),'both','one')\n",
    "\n",
    "df['source'] = df.apply(lambda x: get_origin(x,tablecids),axis = 1)\n",
    "print(df.shape)\n",
    "print(df['source'].value_counts())\n",
    "\"\"\"(36499, 5)\n",
    "both         33139\n",
    "webscrape     3295\n",
    "api             65\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6c7e9548-2c4b-4995-9746-56991083e096",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.to_csv(mainpath + 'output/1_caseidx.csv',sep='\\t',index=False,quoting=csv.QUOTE_ALL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}