data_join.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "56470134-d0fc-44ca-8bcb-60e4f37e3041",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import krippendorff\n",
    "from scipy.stats import spearmanr\n",
    "from itertools import combinations\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "491c4cc9-5c70-4ac9-8dfc-d8765f87be23",
   "metadata": {},
   "source": [
    "Load uses, instances, and judments of a specific round"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ecabebf2-cc82-47be-a49c-11085858f4c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_uses(filename='TRoTR/data/uses.tsv', sep='\\t'):\n",
    "    tmp = list()\n",
    "    with open(filename, mode='r', encoding='utf-8') as f:\n",
    "        columns = f.readline().rstrip().split(sep)\n",
    "        for line in f.readlines():\n",
    "            tmp.append(dict(zip(columns, line.rstrip().split(sep))))\n",
    "    \n",
    "    return pd.DataFrame(tmp)\n",
    "\n",
    "def load_instances(filename, dirname='TRoTR/rounds', sep='\\t'):\n",
    "    tmp = list()\n",
    "    with open(f'{dirname}/{filename}', mode='r', encoding='utf-8') as f:\n",
    "        columns = f.readline().rstrip().split(sep) + ['dataID1', 'dataID2']\n",
    "        for line in f.readlines():\n",
    "            tmp_record = dict(zip(columns, line[:-1].split('\\t')))\n",
    "            tmp_record['dataID1'], tmp_record['dataID2'] = tmp_record['dataIDs'].split(',')\n",
    "            tmp.append(tmp_record)\n",
    "    \n",
    "    return pd.DataFrame(tmp)\n",
    "\n",
    "def load_judgments(filename, dirname='TRoTR/judgments', sep='\\t'):\n",
    "    tmp = list()\n",
    "    with open(f'{dirname}/{filename}', mode='r', encoding='utf-8') as f:\n",
    "        columns = f.readline().rstrip().split(sep)\n",
    "        for line in f.readlines():\n",
    "            tmp_record = dict(zip(columns, line.rstrip().split(sep)))\n",
    "            tmp.append(tmp_record)\n",
    "            \n",
    "    #tmp - to remove after fixing the bug\n",
    "    tmp = list()\n",
    "    with open(f'{dirname}/{filename}', mode='r', encoding='utf-8') as f:\n",
    "        columns = f.readline().rstrip().split(sep)\n",
    "        f = f.read().replace('iosakwe\\n', 'iosakwe@@@').replace('shur\\n', 'shur@@@').replace('Nisha\\n', 'Nisha@@@').replace('AndreaMariaC\\n', 'AndreaMariaC@@@').replace('\\n', '--')\n",
    "        lines = f.split('@@@')\n",
    "        for line in lines:\n",
    "            tmp_record = dict(zip(columns, line.rstrip().split(sep)))\n",
    "            tmp.append(tmp_record)\n",
    "\n",
    "    # -1: can not decide\n",
    "    df = pd.DataFrame(tmp).fillna('-1')\n",
    "    df['label'] = df['label'].apply(lambda x: x.replace('-', '-1')).astype(int)\n",
    "    \n",
    "    return df\n",
    "\n",
    "def merge_data(df_uses, df_instances, df_judgments):\n",
    "    df = df_judgments.merge(df_instances).merge(df_uses, left_on='dataID1', right_on='dataID')\n",
    "    del df['dataID']\n",
    "    del df['lemma']\n",
    "    df = df.rename(columns={column: f'{column}1' for column in ['context', 'indices_target_token', 'indices_target_sentence']})\n",
    "    df = df.merge(df_uses, left_on='dataID2', right_on='dataID')\n",
    "    del df['dataID']\n",
    "    df = df.rename(columns={column: f'{column}2' for column in ['context', 'indices_target_token', 'indices_target_sentence']})\n",
    "    \n",
    "    column_order = ['instanceID', 'dataID1', 'dataID2', 'label', 'annotator',  'lemma', 'context1', 'context2', 'indices_target_token1', 'indices_target_sentence1', 'indices_target_sentence2', 'indices_target_token2',  'comment', 'label_set', 'non_label', 'dataIDs']\n",
    "    return df[column_order]\n",
    "\n",
    "round_ = 'TRoTR.tsv'\n",
    "df_uses = load_uses()\n",
    "df_instances = load_instances(round_)\n",
    "df_judgments = load_judgments(round_)\n",
    "df = merge_data(df_uses, df_instances, df_judgments)\n",
    "\n",
    "def std_judgments(df, lemma=False):\n",
    "    dfs = list()\n",
    "\n",
    "    if not lemma:\n",
    "        for ann in df.annotator.unique():\n",
    "            tmp = df[df['annotator']==ann].copy()\n",
    "            avg = np.mean(tmp.label[tmp.label >= 1].values)\n",
    "            tmp.label = [i if i!=-1 else avg for i in tmp.label]\n",
    "            tmp.label = (tmp.label - tmp.label.mean())/(tmp.label.std())\n",
    "            dfs.append(tmp)\n",
    "        return pd.concat(dfs)\n",
    "    if lemma:\n",
    "        for ann in df.annotator.unique():\n",
    "            for lemma in df.lemma.unique():\n",
    "                tmp = df[(df['annotator']==ann) & (df['lemma']==lemma)].copy()\n",
    "                avg = np.mean(tmp.label[tmp.label >= 1].values)\n",
    "                tmp.label = [i if i!=-1 else avg for i in tmp.label]\n",
    "                tmp.label = (tmp.label - tmp.label.mean())/(tmp.label.std())\n",
    "                dfs.append(tmp)\n",
    "        return pd.concat(dfs)\n",
    "\n",
    "df = std_judgments(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db5b4a76-7717-41f8-bffe-3184b669e068",
   "metadata": {},
   "source": [
    "Group judgments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b6025481-5b80-4187-8f48-655ba722499f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>instanceID</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lemma</th>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">But I suffer not a woman to teach, nor to usurp authority</th>\n",
       "      <th>-1</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>You shall have no other gods before me</th>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">the truth will set you free</th>\n",
       "      <th>1</th>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>65 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                          instanceID\n",
       "lemma                                              label            \n",
       "But I suffer not a woman to teach, nor to usurp... -1              2\n",
       "                                                    1             23\n",
       "                                                    2             77\n",
       "                                                    3            186\n",
       "                                                    4            161\n",
       "...                                                              ...\n",
       "You shall have no other gods before me              4             11\n",
       "the truth will set you free                         1            135\n",
       "                                                    2            192\n",
       "                                                    3             61\n",
       "                                                    4             10\n",
       "\n",
       "[65 rows x 1 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['lemma', 'label']).count()[['instanceID']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d8861a6d-86a4-4c78-a3fa-9af8c72b053d",
   "metadata": {},
   "outputs": [],
   "source": [
    "annotators = ['AndreaMariaC', 'Nisha', 'shur']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "880b5b39-4c24-4480-8b12-40c847a243c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def judgment4annotator(df, annotators):\n",
    "    judgments = defaultdict(lambda: defaultdict(list))\n",
    "    aggregate_agreement = defaultdict(list)\n",
    "    aggregate_other_agreement = defaultdict(lambda: defaultdict(list))\n",
    "    disagreement_judgments = defaultdict(lambda: defaultdict(list))\n",
    "    \n",
    "    verbose = False\n",
    "    for i, row in df[['instanceID']].drop_duplicates().iterrows():\n",
    "        try:\n",
    "            lemma = row['instanceID'].split(\"_\")[2]\n",
    "            \n",
    "            instance_judments = list()\n",
    "            for ann in annotators:\n",
    "                jud = df[(df['instanceID'] == row['instanceID']) & (df['annotator'] == ann)].label.values[0]\n",
    "                instance_judments.append(jud)\n",
    "            \n",
    "            if len(set(instance_judments)) == len(instance_judments):\n",
    "                tmp = df[(df['instanceID'] == row['instanceID']) & (df['annotator'] == annotators[0])]\n",
    "                \n",
    "                if verbose:\n",
    "                    print(tmp.context1.values[0])\n",
    "                    print(tmp.context2.values[0])\n",
    "                    print(row['instanceID'], \" \".join(instance_judments), '\\n')\n",
    "    \n",
    "                for k, ann in enumerate(annotators):\n",
    "                    disagreement_judgments[ann]['all'].append(instance_judments[k])\n",
    "                    disagreement_judgments[ann][lemma].append(instance_judments[k])\n",
    "                #continue\n",
    "    \n",
    "            for k, ann in enumerate(annotators):\n",
    "                judgments[ann]['all'].append(instance_judments[k])\n",
    "                judgments[ann][lemma].append(instance_judments[k])\n",
    "                tmp = [j for a, j in enumerate(instance_judments) if j > -1 and annotators[a]!=ann]\n",
    "                aggregate_other_agreement[ann]['all'].append(round(sum(tmp)/len(tmp)))\n",
    "                aggregate_other_agreement[ann][lemma].append(round(sum(tmp)/len(tmp)))\n",
    "    \n",
    "            # avoid 'cannot decide'\n",
    "            instance_judments = [j for j in instance_judments if j > -1]\n",
    "            aggregate_agreement['all'].append(round(sum(instance_judments)/len(instance_judments)))\n",
    "            aggregate_agreement[lemma].append(round(sum(instance_judments)/len(instance_judments)))\n",
    "        except:\n",
    "            pass\n",
    "                    \n",
    "\n",
    "    return judgments, aggregate_agreement, disagreement_judgments, aggregate_other_agreement\n",
    "\n",
    "judgments, aggregate_agreement, disagreement_judgments, aggregate_other_agreement = judgment4annotator(df, annotators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ac4f3d76-5705-442e-a942-45335a9a1836",
   "metadata": {},
   "outputs": [],
   "source": [
    "def judgment4annotator(df, annotators):\n",
    "    judgments = defaultdict(lambda: defaultdict(list))\n",
    "    aggregate_agreement = defaultdict(list)\n",
    "    disagreement_judgments = defaultdict(lambda: defaultdict(list))\n",
    "    \n",
    "    verbose = False\n",
    "    for i, row in df[['instanceID']].drop_duplicates().iterrows():\n",
    "        try:\n",
    "            lemma = row['instanceID'].split(\"_\")[2]\n",
    "            \n",
    "            instance_judments = list()\n",
    "            for ann in annotators:\n",
    "                jud = df[(df['instanceID'] == row['instanceID']) & (df['annotator'] == ann)].label.values[0]\n",
    "                instance_judments.append(jud)\n",
    "            \n",
    "            if len(set(instance_judments)) == len(instance_judments):\n",
    "                tmp = df[(df['instanceID'] == row['instanceID']) & (df['annotator'] == annotators[0])]\n",
    "                \n",
    "                if verbose:\n",
    "                    print(tmp.context1.values[0])\n",
    "                    print(tmp.context2.values[0])\n",
    "                    print(row['instanceID'], \" \".join(instance_judments), '\\n')\n",
    "    \n",
    "                for k, ann in enumerate(annotators):\n",
    "                    disagreement_judgments[ann]['all'].append(instance_judments[k])\n",
    "                    disagreement_judgments[ann][lemma].append(len(disagreement_judgments[ann]['all'])-1)\n",
    "                #continue\n",
    "    \n",
    "            for k, ann in enumerate(annotators):\n",
    "                judgments[ann]['all'].append(instance_judments[k])\n",
    "                judgments[ann][lemma].append(len(judgments[ann]['all'])-1)\n",
    "    \n",
    "            # avoid 'cannot decide'\n",
    "            instance_judments = [j for j in instance_judments if j > -1]\n",
    "            aggregate_agreement['all'].append(round(sum(instance_judments)/len(instance_judments)))\n",
    "            aggregate_agreement[lemma].append(len(aggregate_agreement['all'])-1)\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    for ann in judgments:\n",
    "        judgments[ann]['all'] = np.array(judgments[ann]['all'])\n",
    "        judgments[ann]['all'] = (judgments[ann]['all'] - judgments[ann]['all'].mean())/(judgments[ann]['all'].std())\n",
    "\n",
    "    for ann in judgments:\n",
    "        for lemma in judgments[ann]:\n",
    "            if lemma == 'all': continue\n",
    "            disagreement_judgments[ann][lemma] = np.array([disagreement_judgments[ann]['all'][i] for i in disagreement_judgments[ann][lemma]])\n",
    "            judgments[ann][lemma] = np.array([judgments[ann]['all'][i] for i in judgments[ann][lemma]])\n",
    "\n",
    "    for lemma in aggregate_agreement:\n",
    "        if lemma == 'all': continue\n",
    "        aggregate_agreement[lemma] = np.array([aggregate_agreement['all'][i] for i in aggregate_agreement[lemma]])\n",
    "        \n",
    "\n",
    "    return judgments, aggregate_agreement, disagreement_judgments\n",
    "\n",
    "judgments, aggregate_agreement, disagreement_judgments = judgment4annotator(df, annotators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "eddb8e63-f60f-4752-a6ab-d314aa236fc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmas = list(judgments[annotators[1]].keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "271cccea-f86b-4ccc-a839-82bc517164ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of pair (all): 1833\n",
      "Number of pair ((Hosea 8:7)): 134\n",
      "Number of pair ((1 Timothy 2:12)): 149\n",
      "Number of pair ((Matthew 7:1)): 89\n",
      "Number of pair ((1 Corinthians 13:4)): 149\n",
      "Number of pair ((Ephesians 5:25)): 142\n",
      "Number of pair ((Jeremiah 17:9)): 122\n",
      "Number of pair ((Exodus 20:3)): 131\n",
      "Number of pair ((2 Corinthians 5:17)): 124\n",
      "Number of pair ((Hebrews 11:1)): 136\n",
      "Number of pair ((1 Samuel 16:7)): 134\n",
      "Number of pair ((Genesis 1:1)): 136\n",
      "Number of pair ((John 8:32)): 103\n",
      "Number of pair ((1 John 4:8)): 150\n",
      "Number of pair ((2 Corinthians 5:7)): 134\n"
     ]
    }
   ],
   "source": [
    "for lemma in lemmas:\n",
    "    print(f'Number of pair ({lemma}):', len(judgments[annotators[1]][lemma]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ce96ce7f-515c-4748-b598-90e43c7dfd1e",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (3,) + inhomogeneous part.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[11], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m lemma \u001b[38;5;129;01min\u001b[39;00m lemmas:\n\u001b[1;32m----> 2\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mKrippendorff Ordinal (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlemma\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m):\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;28mround\u001b[39m(krippendorff\u001b[38;5;241m.\u001b[39malpha(\u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43marray\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mjudgments\u001b[49m\u001b[43m[\u001b[49m\u001b[43mann\u001b[49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[43mlemma\u001b[49m\u001b[43m]\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mann\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mjudgments\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m, level_of_measurement\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mordinal\u001b[39m\u001b[38;5;124m'\u001b[39m), \u001b[38;5;241m3\u001b[39m))\n",
      "\u001b[1;31mValueError\u001b[0m: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (3,) + inhomogeneous part."
     ]
    }
   ],
   "source": [
    "for lemma in lemmas:\n",
    "    print(f'Krippendorff Ordinal ({lemma}):', round(krippendorff.alpha(np.array([judgments[ann][lemma] for ann in judgments]), level_of_measurement='ordinal'), 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "99edf41a-5d35-4bb6-a52f-13ff7dba506c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Krippendorff Interval (all): 0.5\n",
      "Krippendorff Interval ((Hosea 8:7)): 0.275\n",
      "Krippendorff Interval ((1 Timothy 2:12)): 0.339\n",
      "Krippendorff Interval ((Matthew 7:1)): 0.537\n",
      "Krippendorff Interval ((1 Corinthians 13:4)): 0.349\n",
      "Krippendorff Interval ((Ephesians 5:25)): 0.446\n",
      "Krippendorff Interval ((Jeremiah 17:9)): 0.403\n",
      "Krippendorff Interval ((Exodus 20:3)): 0.362\n",
      "Krippendorff Interval ((2 Corinthians 5:17)): 0.458\n",
      "Krippendorff Interval ((Hebrews 11:1)): 0.266\n",
      "Krippendorff Interval ((1 Samuel 16:7)): 0.49\n",
      "Krippendorff Interval ((Genesis 1:1)): 0.247\n",
      "Krippendorff Interval ((John 8:32)): 0.445\n",
      "Krippendorff Interval ((1 John 4:8)): 0.322\n",
      "Krippendorff Interval ((2 Corinthians 5:7)): 0.39\n"
     ]
    }
   ],
   "source": [
    "for lemma in lemmas:\n",
    "    print(f'Krippendorff Interval ({lemma}):', round(krippendorff.alpha(np.array([judgments[ann][lemma] for ann in judgments]), level_of_measurement='interval'), 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6886aea0-6700-45a3-bacc-baccf442bc3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- all --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.57 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.526 (pvalue=0.0)\n",
      "Spearman corr. (shur, agg): 0.471 (pvalue=0.0)\n",
      "-- (Ephesians 5:25) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.54 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.594 (pvalue=0.0)\n",
      "Spearman corr. (shur, agg): 0.559 (pvalue=0.0)\n",
      "-- (Jeremiah 17:9) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.51 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.407 (pvalue=0.0)\n",
      "Spearman corr. (shur, agg): 0.378 (pvalue=0.0)\n",
      "-- (Exodus 20:3) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.471 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.383 (pvalue=0.0)\n",
      "Spearman corr. (shur, agg): 0.371 (pvalue=0.0)\n",
      "-- (2 Corinthians 5:17) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.509 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.481 (pvalue=0.0)\n",
      "Spearman corr. (shur, agg): 0.497 (pvalue=0.0)\n",
      "-- (Hebrews 11:1) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.453 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.493 (pvalue=0.0)\n",
      "Spearman corr. (shur, agg): 0.261 (pvalue=0.012)\n",
      "-- (1 Samuel 16:7) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.498 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.428 (pvalue=0.0)\n",
      "Spearman corr. (shur, agg): 0.51 (pvalue=0.0)\n",
      "-- (Genesis 1:1) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.106 (pvalue=0.295)\n",
      "Spearman corr. (Nisha, agg): 0.217 (pvalue=0.03)\n",
      "Spearman corr. (shur, agg): 0.091 (pvalue=0.369)\n",
      "-- (John 8:32) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.581 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.366 (pvalue=0.0)\n",
      "Spearman corr. (shur, agg): 0.44 (pvalue=0.0)\n",
      "-- (1 John 4:8) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.438 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.341 (pvalue=0.001)\n",
      "Spearman corr. (shur, agg): 0.331 (pvalue=0.001)\n",
      "-- (2 Corinthians 5:7) --\n",
      "Spearman corr. (AndreaMariaC, agg): 0.56 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, agg): 0.393 (pvalue=0.0)\n",
      "Spearman corr. (shur, agg): 0.25 (pvalue=0.017)\n"
     ]
    }
   ],
   "source": [
    "for lemma in lemmas:\n",
    "    print(f'-- {lemma} --')\n",
    "    for k, ann in enumerate(judgments):\n",
    "        corr, pvalue = spearmanr(aggregate_other_agreement[ann][lemma], judgments[ann][lemma])\n",
    "        print(f'Spearman corr. ({annotators[k]}, agg):', round(corr, 3), f'(pvalue={round(pvalue, 3)})')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "bd724bb4-ce9e-433b-a512-2de79c5e68fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- all --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.596 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.491 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.461 (pvalue=0.0)\n",
      "-- (Hosea 8:7) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.474 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.435 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.378 (pvalue=0.0)\n",
      "-- (1 Timothy 2:12) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.47 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.293 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.27 (pvalue=0.001)\n",
      "-- (Matthew 7:1) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.532 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.679 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.455 (pvalue=0.0)\n",
      "-- (1 Corinthians 13:4) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.466 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.39 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.29 (pvalue=0.0)\n",
      "-- (Ephesians 5:25) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.595 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.434 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.565 (pvalue=0.0)\n",
      "-- (Jeremiah 17:9) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.452 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.412 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.388 (pvalue=0.0)\n",
      "-- (Exodus 20:3) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.431 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.39 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.321 (pvalue=0.002)\n",
      "-- (2 Corinthians 5:17) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.43 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.546 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.481 (pvalue=0.0)\n",
      "-- (Hebrews 11:1) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.453 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.297 (pvalue=0.004)\n",
      "Spearman corr. (Nisha, shur): 0.329 (pvalue=0.001)\n",
      "-- (1 Samuel 16:7) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.507 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.552 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.405 (pvalue=0.0)\n",
      "-- (Genesis 1:1) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.194 (pvalue=0.053)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.257 (pvalue=0.01)\n",
      "Spearman corr. (Nisha, shur): 0.259 (pvalue=0.009)\n",
      "-- (John 8:32) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.452 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.535 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.31 (pvalue=0.002)\n",
      "-- (1 John 4:8) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.354 (pvalue=0.001)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.347 (pvalue=0.001)\n",
      "Spearman corr. (Nisha, shur): 0.261 (pvalue=0.012)\n",
      "-- (2 Corinthians 5:7) --\n",
      "Spearman corr. (AndreaMariaC, Nisha): 0.49 (pvalue=0.0)\n",
      "Spearman corr. (AndreaMariaC, shur): 0.438 (pvalue=0.0)\n",
      "Spearman corr. (Nisha, shur): 0.229 (pvalue=0.029)\n"
     ]
    }
   ],
   "source": [
    "for lemma in lemmas:\n",
    "    print(f'-- {lemma} --')\n",
    "    for ann1, ann2 in combinations(annotators, 2):\n",
    "        corr, pvalue = spearmanr(judgments[ann1][lemma], judgments[ann2][lemma])\n",
    "        print(f'Spearman corr. ({ann1}, {ann2}):', round(corr, 3), f'(pvalue={round(pvalue, 3)})')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "1c03f3cd-c2a3-49bf-aee2-5c42b832532e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- all --\n",
      "Class distribution (AndreaMariaC): (array([1, 2, 3, 4]), array([ 57, 350, 297,  55], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3, 4]), array([259, 260, 207,  33], dtype=int64))\n",
      "Class distribution (shur): (array([-1,  1,  2,  3,  4]), array([ 22, 145, 203, 263, 126], dtype=int64))\n",
      "-- (Ephesians 5:25) --\n",
      "Class distribution (AndreaMariaC): (array([1, 2, 3, 4]), array([ 1, 17, 44,  8], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3, 4]), array([12, 20, 34,  4], dtype=int64))\n",
      "Class distribution (shur): (array([-1,  1,  2,  3,  4]), array([ 4,  4, 24, 24, 14], dtype=int64))\n",
      "-- (Jeremiah 17:9) --\n",
      "Class distribution (AndreaMariaC): (array([1, 2, 3, 4]), array([ 9, 39, 23,  6], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3, 4]), array([45, 19,  8,  5], dtype=int64))\n",
      "Class distribution (shur): (array([-1,  1,  2,  3,  4]), array([ 3, 17, 15, 28, 14], dtype=int64))\n",
      "-- (Exodus 20:3) --\n",
      "Class distribution (AndreaMariaC): (array([1, 2, 3, 4]), array([ 3, 43, 18,  3], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3, 4]), array([24, 30, 12,  1], dtype=int64))\n",
      "Class distribution (shur): (array([-1,  1,  2,  3,  4]), array([ 1, 16, 25, 21,  4], dtype=int64))\n",
      "-- (2 Corinthians 5:17) --\n",
      "Class distribution (AndreaMariaC): (array([1, 2, 3, 4]), array([ 8, 36, 28,  8], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3, 4]), array([41, 20, 17,  2], dtype=int64))\n",
      "Class distribution (shur): (array([-1,  1,  2,  3,  4]), array([ 4, 17, 18, 32,  9], dtype=int64))\n",
      "-- (Hebrews 11:1) --\n",
      "Class distribution (AndreaMariaC): (array([1, 2, 3, 4]), array([11, 22, 42,  3], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3, 4]), array([21, 29, 26,  2], dtype=int64))\n",
      "Class distribution (shur): (array([-1,  1,  2,  3,  4]), array([10, 10, 21, 28,  9], dtype=int64))\n",
      "-- (1 Samuel 16:7) --\n",
      "Class distribution (AndreaMariaC): (array([1, 2, 3, 4]), array([11, 44, 26,  8], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3, 4]), array([32, 28, 26,  3], dtype=int64))\n",
      "Class distribution (shur): (array([1, 2, 3, 4]), array([13, 33, 29, 14], dtype=int64))\n",
      "-- (Genesis 1:1) --\n",
      "Class distribution (AndreaMariaC): (array([1, 2, 3, 4]), array([ 3, 54, 22,  2], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3]), array([20, 48, 13], dtype=int64))\n",
      "Class distribution (shur): (array([1, 2, 3, 4]), array([19, 34, 23,  5], dtype=int64))\n",
      "-- (John 8:32) --\n",
      "Class distribution (AndreaMariaC): (array([1, 2, 3, 4]), array([11, 57, 12,  1], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3]), array([45, 29,  7], dtype=int64))\n",
      "Class distribution (shur): (array([1, 2, 3, 4]), array([34, 23, 17,  7], dtype=int64))\n",
      "-- (1 John 4:8) --\n",
      "Class distribution (AndreaMariaC): (array([2, 3, 4]), array([ 9, 51, 10], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3, 4]), array([ 2, 11, 44, 13], dtype=int64))\n",
      "Class distribution (shur): (array([1, 2, 3, 4]), array([ 1,  5, 36, 28], dtype=int64))\n",
      "-- (2 Corinthians 5:7) --\n",
      "Class distribution (AndreaMariaC): (array([2, 3, 4]), array([29, 31,  6], dtype=int64))\n",
      "Class distribution (Nisha): (array([1, 2, 3, 4]), array([17, 26, 20,  3], dtype=int64))\n",
      "Class distribution (shur): (array([1, 2, 3, 4]), array([14,  5, 25, 22], dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "for lemma in lemmas:\n",
    "    print(f'-- {lemma} --')\n",
    "    for ann in annotators:\n",
    "        print(f'Class distribution ({ann}):', np.unique(np.array(judgments[ann][lemma]), return_counts=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8111d027-39c8-422a-b19f-641c5232e3ed",
   "metadata": {},
   "source": [
    "# Quality Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "d865a0dc-35b1-44b4-b3b3-a50aa166dd1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "round_ = 'post-1st-round.tsv'\n",
    "\n",
    "df_instances_post1st = load_instances(round_)\n",
    "df_judgments_post1st = load_judgments(round_)\n",
    "df_post1st = merge_data(df_uses, df_instances_post1st, df_judgments_post1st)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "ce9ed3e3-8706-4e4e-a4ec-4fb18811c3ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "annotators = ['AndreaMariaC', 'shur', 'Nisha']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "8ec53320-dfb7-47fc-bdd1-72543e8690b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1st = df[df['instanceID'].isin(df_post1st.instanceID.values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "8ed1e87c-00d1-40fc-b40e-05f9942d9e3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_post1st = df_1st[['instanceID', 'annotator', 'label']].merge(df_post1st[['instanceID', 'annotator', 'label']], left_on=['instanceID', 'annotator'], right_on=['instanceID', 'annotator'])\n",
    "df_post1st['diff'] = np.abs(df_post1st['label_x'].values - df_post1st['label_y'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "c8b88e72-cc8e-4c5e-ac2e-bf01e8cfe2bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spearman corr. (AndreaMariaC): 0.448 (pvalue=0.042)\n",
      "Spearman corr. (shur): 0.546 (pvalue=0.01)\n",
      "Spearman corr. (Nisha): 0.415 (pvalue=0.061)\n"
     ]
    }
   ],
   "source": [
    "for ann in annotators:\n",
    "    corr, pvalue = spearmanr(df_post1st[df_post1st['annotator'] == ann].label_x.values, df_post1st[df_post1st['annotator'] == ann].label_y.values)\n",
    "    print(f'Spearman corr. ({ann}):', round(corr, 3), f'(pvalue={round(pvalue, 3)})')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "37843bda-fafe-49e2-89a6-80629d2a8368",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>instanceID</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>annotator</th>\n",
       "      <th>label_x</th>\n",
       "      <th>diff</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">AndreaMariaC</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">2</th>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"9\" valign=\"top\">Nisha</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">1</th>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">3</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"9\" valign=\"top\">shur</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">1</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">4</th>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           instanceID\n",
       "annotator    label_x diff            \n",
       "AndreaMariaC 1       0              7\n",
       "                     1              2\n",
       "             2       0              2\n",
       "                     1              6\n",
       "                     2              1\n",
       "             3       0              2\n",
       "                     2              1\n",
       "Nisha        1       0              7\n",
       "                     1              3\n",
       "                     2              2\n",
       "             2       0              2\n",
       "                     1              1\n",
       "             3       0              1\n",
       "                     1              3\n",
       "                     2              1\n",
       "             4       1              1\n",
       "shur         1       0              1\n",
       "                     1              1\n",
       "                     2              2\n",
       "             2       0              2\n",
       "                     2              1\n",
       "             3       0              1\n",
       "                     1              7\n",
       "             4       0              5\n",
       "                     1              1"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_post1st.groupby(['annotator', 'label_x', 'diff']).count()[['instanceID']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "213c0feb-2d6d-45c0-b7f4-c7b65d3c9509",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_shur = df[df['annotator'] == 'shur'].reset_index()\n",
    "del df_shur['annotator']\n",
    "df_nisha = df[df['annotator'] == 'Nisha'].reset_index()\n",
    "del df_nisha['annotator']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bc42a629-892c-440f-81f0-e3839ac78a0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sn = df_nisha[['instanceID', 'label']].merge(df_shur[['instanceID', 'label']], left_on=['instanceID'], right_on=['instanceID'])\n",
    "df_sn['diff'] = np.abs(df_sn['label_x'].values - df_sn['label_y'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4d5e8377-53bb-45d5-aa0d-3431975d0f4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>instanceID</th>\n",
       "      <th>label_x</th>\n",
       "      <th>label_y</th>\n",
       "      <th>diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pair_28_(Matthew 7:1)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pair_74_(Matthew 7:1)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pair_4_(Matthew 7:1)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>pair_125_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>pair_85_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pair_121_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>pair_66_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>pair_127_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>pair_53_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>pair_99_(Matthew 7:1)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>pair_29_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>pair_82_(1 Corinthians 13:4)</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>pair_107_(1 Corinthians 13:4)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>pair_106_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>pair_72_(1 Corinthians 13:4)</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>pair_52_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>pair_14_(1 Corinthians 13:4)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>pair_43_(1 Corinthians 13:4)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>pair_102_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>pair_22_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>pair_115_(1 Corinthians 13:4)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>pair_132_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>pair_78_(1 Corinthians 13:4)</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>pair_44_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>pair_40_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>pair_99_(1 Corinthians 13:4)</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>pair_75_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>pair_80_(1 Corinthians 13:4)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>pair_118_(1 Corinthians 13:4)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>pair_66_(1 Corinthians 13:4)</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>pair_43_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>pair_94_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>pair_38_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>pair_139_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>pair_73_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>pair_29_(Matthew 7:1)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>pair_122_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>pair_118_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>pair_2_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>pair_87_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>pair_142_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>pair_134_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>pair_97_(Matthew 7:1)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>pair_97_(1 Corinthians 13:4)</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>pair_46_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>pair_121_(1 Corinthians 13:4)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>pair_105_(1 Corinthians 13:4)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>pair_104_(1 Corinthians 13:4)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>pair_62_(Matthew 7:1)</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284</th>\n",
       "      <td>pair_105_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>pair_77_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>pair_13_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>pair_25_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>pair_96_(Matthew 7:1)</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        instanceID  label_x  label_y  diff\n",
       "1            pair_28_(Matthew 7:1)        2        4     2\n",
       "2            pair_74_(Matthew 7:1)        2        4     2\n",
       "3             pair_4_(Matthew 7:1)        2        4     2\n",
       "6           pair_125_(Matthew 7:1)        1        3     2\n",
       "12           pair_85_(Matthew 7:1)        1        3     2\n",
       "13          pair_121_(Matthew 7:1)        1        3     2\n",
       "20           pair_66_(Matthew 7:1)        1        3     2\n",
       "23          pair_127_(Matthew 7:1)        1        3     2\n",
       "37           pair_53_(Matthew 7:1)        1        4     3\n",
       "41           pair_99_(Matthew 7:1)        2        4     2\n",
       "49    pair_29_(1 Corinthians 13:4)        2        4     2\n",
       "54    pair_82_(1 Corinthians 13:4)        4        1     3\n",
       "59   pair_107_(1 Corinthians 13:4)        1        3     2\n",
       "60   pair_106_(1 Corinthians 13:4)        2        4     2\n",
       "62    pair_72_(1 Corinthians 13:4)        4        2     2\n",
       "68    pair_52_(1 Corinthians 13:4)        2        4     2\n",
       "71    pair_14_(1 Corinthians 13:4)        1        3     2\n",
       "72    pair_43_(1 Corinthians 13:4)        1        3     2\n",
       "76   pair_102_(1 Corinthians 13:4)        2        4     2\n",
       "82    pair_22_(1 Corinthians 13:4)        2        4     2\n",
       "85   pair_115_(1 Corinthians 13:4)        1        3     2\n",
       "87   pair_132_(1 Corinthians 13:4)        2        4     2\n",
       "90    pair_78_(1 Corinthians 13:4)        4        1     3\n",
       "94    pair_44_(1 Corinthians 13:4)        2        4     2\n",
       "96    pair_40_(1 Corinthians 13:4)        2        4     2\n",
       "97    pair_99_(1 Corinthians 13:4)        3        1     2\n",
       "100   pair_75_(1 Corinthians 13:4)        2        4     2\n",
       "114   pair_80_(1 Corinthians 13:4)        1        3     2\n",
       "118  pair_118_(1 Corinthians 13:4)        1        3     2\n",
       "144   pair_66_(1 Corinthians 13:4)        3        1     2\n",
       "180          pair_43_(Matthew 7:1)        1        3     2\n",
       "183          pair_94_(Matthew 7:1)        1        3     2\n",
       "188          pair_38_(Matthew 7:1)        1        3     2\n",
       "193         pair_139_(Matthew 7:1)        1        3     2\n",
       "194          pair_73_(Matthew 7:1)        1        3     2\n",
       "195          pair_29_(Matthew 7:1)        2        4     2\n",
       "196         pair_122_(Matthew 7:1)        1        3     2\n",
       "198         pair_118_(Matthew 7:1)        1        4     3\n",
       "201           pair_2_(Matthew 7:1)        1        3     2\n",
       "207          pair_87_(Matthew 7:1)        1        3     2\n",
       "208         pair_142_(Matthew 7:1)        1        3     2\n",
       "212         pair_134_(Matthew 7:1)        1        3     2\n",
       "226          pair_97_(Matthew 7:1)        2        4     2\n",
       "229   pair_97_(1 Corinthians 13:4)        3        1     2\n",
       "232   pair_46_(1 Corinthians 13:4)        2        4     2\n",
       "247  pair_121_(1 Corinthians 13:4)        1        3     2\n",
       "248  pair_105_(1 Corinthians 13:4)        1        3     2\n",
       "266  pair_104_(1 Corinthians 13:4)        2        4     2\n",
       "280          pair_62_(Matthew 7:1)        2        4     2\n",
       "284         pair_105_(Matthew 7:1)        1        3     2\n",
       "288          pair_77_(Matthew 7:1)        1        3     2\n",
       "295          pair_13_(Matthew 7:1)        1        3     2\n",
       "296          pair_25_(Matthew 7:1)        1        4     3\n",
       "297          pair_96_(Matthew 7:1)        1        3     2"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sn[df_sn['diff'] > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "bbd5d37b-e14c-437a-a985-2826746acc80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Exactly. Judge not, that ye be not judged. Is NOT in the Bible. It's a cute coined phrase that sinners use to justify their sin while the meaning of the CONTEXT means to judge those who you don't have a sin in.\""
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['instanceID'] == 'pair_105_(Matthew 7:1)'][['context1', 'context2']].context1.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "175a65bb-954b-491a-9550-380666c27410",
   "metadata": {},
   "outputs": [],
   "source": [
    "Love is eternal, indestructible, and always worth fighting for. Love is patient, love is kind, and love endures forever. Father's rights are God-given.\n",
    "\n",
    "When the love is patient and kind, it is worthy to 'fight' for."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}