stats+DURel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "8f899398-eb18-48f2-ba03-2c45f1089d3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import csv\n",
    "import krippendorff\n",
    "from scipy.stats import spearmanr\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from itertools import combinations\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fe8d41d-76c1-44ec-9804-2fe6583d9a4d",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "77df0f5f-a917-4701-849e-31c1dee6cfd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_uses(filename='TRoTR/data/uses.tsv', sep='\\t'):\n",
    "    df = list()\n",
    "    with open(filename, mode='r', encoding='utf-8') as f:\n",
    "        columns = f.readline().rstrip().split(sep)\n",
    "        for line in f.readlines():\n",
    "            df.append(dict(zip(columns, line.rstrip().split(sep))))\n",
    "    \n",
    "    return pd.DataFrame(df)\n",
    "\n",
    "def load_instances(filename, dirname='TRoTR/rounds', sep='\\t'):\n",
    "    df = list()\n",
    "    with open(f'{dirname}/{filename}', mode='r', encoding='utf-8') as f:\n",
    "        columns = f.readline().rstrip().split(sep) + ['dataID1', 'dataID2']\n",
    "        for line in f.readlines():\n",
    "            record = dict(zip(columns, line[:-1].split('\\t')))\n",
    "            record['dataID1'], record['dataID2'] = record['dataIDs'].split(',')\n",
    "            df.append(record)\n",
    "    \n",
    "    return pd.DataFrame(df)\n",
    "\n",
    "def load_judgments(filename, dirname='TRoTR/judgments', sep='\\t'):\n",
    "    df = list()\n",
    "    with open(f'{dirname}/{filename}', mode='r', encoding='utf-8') as f:\n",
    "        columns = f.readline().rstrip().split(sep)\n",
    "        for line in f.readlines():\n",
    "            record = dict(zip(columns, line.rstrip().split(sep)))\n",
    "            if record['label'] == '-':\n",
    "                record['label'] = math.nan\n",
    "            df.append(record)\n",
    "\n",
    "    df = pd.DataFrame(df)\n",
    "    df['label'] = df['label'].astype(float)\n",
    "    \n",
    "    return df\n",
    "\n",
    "def merge_data(df_uses, df_instances, df_judgments):\n",
    "    df = df_judgments.merge(df_instances).merge(df_uses, left_on='dataID1', right_on='dataID')\n",
    "    del df['dataID']\n",
    "    del df['lemma']\n",
    "    df = df.rename(columns={column: f'{column}1' for column in ['context', 'indices_target_token', 'indices_target_sentence']})\n",
    "    df = df.merge(df_uses, left_on='dataID2', right_on='dataID')\n",
    "    del df['dataID']\n",
    "    df = df.rename(columns={column: f'{column}2' for column in ['context', 'indices_target_token', 'indices_target_sentence']})\n",
    "    \n",
    "    column_order = ['instanceID', 'dataID1', 'dataID2', 'label', 'annotator',  'lemma', 'context1', 'context2', 'indices_target_token1', 'indices_target_sentence1', 'indices_target_sentence2', 'indices_target_token2',  'comment', 'label_set', 'non_label', 'dataIDs']\n",
    "    return df[column_order]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "97bb7852-4842-437c-b408-4d3376e495c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "round_ = 'TRoTR.tsv'\n",
    "df_judgments = load_judgments(round_)\n",
    "df_uses = load_uses()\n",
    "df_instances = load_instances(round_)\n",
    "df = merge_data(df_uses, df_instances, df_judgments)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffd13c2f-b88e-489e-ad62-9638b1a84667",
   "metadata": {},
   "source": [
    "# Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "9720b5df-2482-488e-a618-59d0e595d7cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def inter_annotator_agreement(df, targets=None, instances=None):\n",
    "    targets = df.lemma.unique() if targets is None else targets\n",
    "    instances = df.instanceID.unique() if instances is None else instances\n",
    "    df = df[df['lemma'].isin(targets) & df['instanceID'].isin(instances)]\n",
    "    \n",
    "    pairwise_spearman = list()\n",
    "    weights = list()\n",
    "    \n",
    "    annotators = df.annotator.unique()\n",
    "    for annotator1 in annotators:\n",
    "        for annotator2 in annotators:\n",
    "            if annotator1 == annotator2: continue\n",
    "            \n",
    "            df1 = df[df['annotator'] == annotator1]\n",
    "            df2 = df[df['annotator'] == annotator2]\n",
    "            \n",
    "            if df2.shape[0] > df1.shape[0]:\n",
    "                df1, df2 = df2, df1\n",
    "    \n",
    "            instances=df2.instanceID.values\n",
    "            df1=df1[df1['instanceID'].isin(instances)].sort_values('instanceID')\n",
    "            df2=df2[df2['instanceID'].isin(instances)].sort_values('instanceID')\n",
    "            corr, pvalue = spearmanr(df1.label.values, df2.label.values, nan_policy='omit')\n",
    "            pairwise_spearman.append(corr)\n",
    "            weights.append(instances.shape[0])\n",
    "\n",
    "    return np.average(pairwise_spearman, weights=weights).round(3)\n",
    "\n",
    "def krippendorff_agreement(df, targets=None, instances=None):\n",
    "    targets = df.lemma.unique() if targets is None else targets\n",
    "    instances = df.instanceID.unique() if instances is None else instances\n",
    "    df = df[df['lemma'].isin(targets) & df['instanceID'].isin(instances)]\n",
    "\n",
    "    instances = df.instanceID.unique()\n",
    "    annotators = df.annotator.unique()\n",
    "    \n",
    "    judgments = list()\n",
    "    for annotator in annotators:\n",
    "        df_ann = df[df['annotator']==annotator].sort_values('instanceID')\n",
    "\n",
    "        # add nan values\n",
    "        if df_ann.shape[0] < instances.shape[0]:\n",
    "            nan_instances = np.setdiff1d(instances, df_ann.instanceID.values)\n",
    "            new_df_ann = pd.DataFrame()\n",
    "            new_df_ann['instanceID'] = nan_instances\n",
    "            new_df_ann['label'] = math.nan\n",
    "            df_ann = pd.concat([df_ann, new_df_ann]).sort_values('instanceID')\n",
    "        \n",
    "        judgments.append(df_ann.label.values)\n",
    "\n",
    "    return round(krippendorff.alpha(judgments, level_of_measurement='ordinal'), 3)\n",
    "\n",
    "def inter_annotator_agreement_per_target(df, targets=None, instances=None):\n",
    "    targets = df.lemma.unique() if targets is None else targets\n",
    "    instances = df.instanceID.unique() if instances is None else instances\n",
    "    df = df[df['lemma'].isin(targets) & df['instanceID'].isin(instances)]\n",
    "    \n",
    "    pairwise_spearman = defaultdict(list)\n",
    "    weights = defaultdict(list)\n",
    "    \n",
    "    annotators = df.annotator.unique()\n",
    "    targets = df.lemma.unique()\n",
    "    instances_dict = defaultdict(int)\n",
    "    for target in targets:\n",
    "        for annotator1 in annotators:\n",
    "            for annotator2 in annotators:\n",
    "                if annotator1 == annotator2: continue\n",
    "                    \n",
    "                df1 = df[(df['annotator'] == annotator1) & (df['lemma'] == target)]\n",
    "                df2 = df[(df['annotator'] == annotator2)  & (df['lemma'] == target)]\n",
    "                \n",
    "                if df2.shape[0] > df1.shape[0]:\n",
    "                    df1, df2 = df2, df1\n",
    "        \n",
    "                instances=df2.instanceID.values\n",
    "                instances_dict[target]=max(instances.shape[0], instances_dict[target])\n",
    "                df1=df1[df1['instanceID'].isin(instances)].sort_values('instanceID')\n",
    "                df2=df2[df2['instanceID'].isin(instances)].sort_values('instanceID')\n",
    "                corr, pvalue = spearmanr(df1.label.values, df2.label.values, nan_policy='omit')\n",
    "                if corr == corr: # != math.nan\n",
    "                    pairwise_spearman[target].append(corr)\n",
    "                    weights[target].append(instances.shape[0])\n",
    "\n",
    "    df_res = pd.DataFrame()\n",
    "    df_res['lemma'] = targets\n",
    "    df_res['avg_pairwise_spearman_agreement'] = [np.average(pairwise_spearman[target], weights=weights[target]).round(3) for target in targets]\n",
    "    df_res['n_instances'] = [instances_dict[target] for target in targets]\n",
    "\n",
    "    return df_res\n",
    "\n",
    "def krippendorff_agreement_per_target(df, targets=None, instances=None):\n",
    "    targets = df.lemma.unique() if targets is None else targets\n",
    "    instances = df.instanceID.unique() if instances is None else instances\n",
    "    df = df[df['lemma'].isin(targets) & df['instanceID'].isin(instances)]\n",
    "    \n",
    "    targets = df.lemma.unique()\n",
    "    annotators = df.annotator.unique()\n",
    "\n",
    "    df_res = list()\n",
    "    for target in targets:\n",
    "        instances = df[df['lemma']==target].instanceID.unique()\n",
    "        \n",
    "        judgments = list()\n",
    "        for annotator in annotators:\n",
    "            df_ann = df[(df['annotator']==annotator) & (df['instanceID'].isin(instances))].sort_values('instanceID')\n",
    "    \n",
    "            # add nan values\n",
    "            if df_ann.shape[0] < instances.shape[0]:\n",
    "                nan_instances = np.setdiff1d(instances, df_ann.instanceID.values)\n",
    "                new_df_ann = pd.DataFrame()\n",
    "                new_df_ann['instanceID'] = nan_instances\n",
    "                new_df_ann['label'] = math.nan\n",
    "                df_ann = pd.concat([df_ann, new_df_ann]).sort_values('instanceID')\n",
    "            \n",
    "            judgments.append(df_ann.label.values)\n",
    "\n",
    "        df_res.append(dict(lemma=target, krippendorff_agreement=round(krippendorff.alpha(judgments, level_of_measurement='ordinal'), 3), n_instances=instances.shape[0]))\n",
    "\n",
    "    return pd.DataFrame(df_res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "96893f52-771d-4a6e-9d96-8b7dd1a58cb4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.506, 0.42)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aps = inter_annotator_agreement(df) # average pairwise spearman correlation\n",
    "ka = krippendorff_agreement(df)\n",
    "(aps, ka)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "fa504271-a677-47c3-baa6-a72407c5276c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lemma</th>\n",
       "      <th>krippendorff_agreement</th>\n",
       "      <th>n_instances_x</th>\n",
       "      <th>avg_pairwise_spearman_agreement</th>\n",
       "      <th>n_instances_y</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>that all may be one</td>\n",
       "      <td>0.118</td>\n",
       "      <td>150</td>\n",
       "      <td>0.183</td>\n",
       "      <td>150</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Seventy times seven</td>\n",
       "      <td>0.494</td>\n",
       "      <td>150</td>\n",
       "      <td>0.619</td>\n",
       "      <td>150</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>turn the other cheek</td>\n",
       "      <td>-0.036</td>\n",
       "      <td>150</td>\n",
       "      <td>0.510</td>\n",
       "      <td>150</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Seek and you will find</td>\n",
       "      <td>0.210</td>\n",
       "      <td>150</td>\n",
       "      <td>0.558</td>\n",
       "      <td>150</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>The Lord is my shepherd</td>\n",
       "      <td>0.213</td>\n",
       "      <td>150</td>\n",
       "      <td>0.476</td>\n",
       "      <td>150</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>the truth will set you free</td>\n",
       "      <td>0.250</td>\n",
       "      <td>150</td>\n",
       "      <td>0.347</td>\n",
       "      <td>150</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Love is patient, love is kind</td>\n",
       "      <td>0.282</td>\n",
       "      <td>150</td>\n",
       "      <td>0.382</td>\n",
       "      <td>150</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>For everything there is a season</td>\n",
       "      <td>0.263</td>\n",
       "      <td>150</td>\n",
       "      <td>0.443</td>\n",
       "      <td>150</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Judge not, that ye be not judged</td>\n",
       "      <td>0.472</td>\n",
       "      <td>150</td>\n",
       "      <td>0.555</td>\n",
       "      <td>150</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>all things work together for good</td>\n",
       "      <td>0.110</td>\n",
       "      <td>150</td>\n",
       "      <td>0.543</td>\n",
       "      <td>150</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>For we walk by faith, not by sight</td>\n",
       "      <td>0.383</td>\n",
       "      <td>150</td>\n",
       "      <td>0.437</td>\n",
       "      <td>150</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>The Lord will keep you from all harm</td>\n",
       "      <td>0.169</td>\n",
       "      <td>150</td>\n",
       "      <td>0.213</td>\n",
       "      <td>150</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Give to Caesar what belongs to Caesar</td>\n",
       "      <td>0.196</td>\n",
       "      <td>150</td>\n",
       "      <td>0.460</td>\n",
       "      <td>150</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Better is open rebuke than hidden love</td>\n",
       "      <td>0.431</td>\n",
       "      <td>150</td>\n",
       "      <td>0.481</td>\n",
       "      <td>150</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>You shall have no other gods before me</td>\n",
       "      <td>0.259</td>\n",
       "      <td>150</td>\n",
       "      <td>0.331</td>\n",
       "      <td>150</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>In the beginning God created the heaven</td>\n",
       "      <td>0.151</td>\n",
       "      <td>150</td>\n",
       "      <td>0.219</td>\n",
       "      <td>150</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>If a man lies with a male as with a woman</td>\n",
       "      <td>0.315</td>\n",
       "      <td>150</td>\n",
       "      <td>0.339</td>\n",
       "      <td>150</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Love one another with brotherly affection</td>\n",
       "      <td>0.410</td>\n",
       "      <td>150</td>\n",
       "      <td>0.571</td>\n",
       "      <td>150</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Be strong and courageous. Do not be afraid</td>\n",
       "      <td>0.223</td>\n",
       "      <td>150</td>\n",
       "      <td>0.281</td>\n",
       "      <td>150</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Everything is possible for one who believes</td>\n",
       "      <td>0.081</td>\n",
       "      <td>150</td>\n",
       "      <td>0.118</td>\n",
       "      <td>150</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>I can do all things through Christ who strengt...</td>\n",
       "      <td>0.128</td>\n",
       "      <td>150</td>\n",
       "      <td>0.349</td>\n",
       "      <td>150</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Husbands, love your wives, as Christ loved the...</td>\n",
       "      <td>0.487</td>\n",
       "      <td>150</td>\n",
       "      <td>0.507</td>\n",
       "      <td>150</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Love your enemies and pray for those who perse...</td>\n",
       "      <td>0.073</td>\n",
       "      <td>150</td>\n",
       "      <td>0.389</td>\n",
       "      <td>150</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>My command is this: Love each other as I have ...</td>\n",
       "      <td>0.288</td>\n",
       "      <td>150</td>\n",
       "      <td>0.426</td>\n",
       "      <td>150</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>you are precious in my eyes and honored, and I...</td>\n",
       "      <td>0.421</td>\n",
       "      <td>150</td>\n",
       "      <td>0.625</td>\n",
       "      <td>150</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>The rain came down, the streams rose, and the ...</td>\n",
       "      <td>0.166</td>\n",
       "      <td>150</td>\n",
       "      <td>0.406</td>\n",
       "      <td>150</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Hatred stirs up conflict, but love covers over...</td>\n",
       "      <td>0.172</td>\n",
       "      <td>150</td>\n",
       "      <td>0.377</td>\n",
       "      <td>150</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>But I suffer not a woman to teach, nor to usur...</td>\n",
       "      <td>0.302</td>\n",
       "      <td>150</td>\n",
       "      <td>0.315</td>\n",
       "      <td>150</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>They have sown the wind, and they shall reap t...</td>\n",
       "      <td>0.261</td>\n",
       "      <td>150</td>\n",
       "      <td>0.423</td>\n",
       "      <td>150</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Anxiety weighs down the heart, but a kind word...</td>\n",
       "      <td>0.093</td>\n",
       "      <td>150</td>\n",
       "      <td>0.253</td>\n",
       "      <td>150</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Whoever does not love does not know God, becau...</td>\n",
       "      <td>0.329</td>\n",
       "      <td>150</td>\n",
       "      <td>0.373</td>\n",
       "      <td>150</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>You are altogether beautiful, my darling; ther...</td>\n",
       "      <td>0.385</td>\n",
       "      <td>150</td>\n",
       "      <td>0.537</td>\n",
       "      <td>150</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>You shall not lie with a male as with a woman;...</td>\n",
       "      <td>0.423</td>\n",
       "      <td>150</td>\n",
       "      <td>0.458</td>\n",
       "      <td>150</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>This is the day that the Lord has made; let us...</td>\n",
       "      <td>0.294</td>\n",
       "      <td>150</td>\n",
       "      <td>0.492</td>\n",
       "      <td>150</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>A wife of noble character who can find? She is...</td>\n",
       "      <td>0.108</td>\n",
       "      <td>150</td>\n",
       "      <td>0.261</td>\n",
       "      <td>150</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Greater love has no one than this: to lay down...</td>\n",
       "      <td>0.347</td>\n",
       "      <td>150</td>\n",
       "      <td>0.640</td>\n",
       "      <td>150</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Come to me, all you who labour and are overbur...</td>\n",
       "      <td>-0.007</td>\n",
       "      <td>150</td>\n",
       "      <td>0.268</td>\n",
       "      <td>150</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>The heart is deceitful above all things, and d...</td>\n",
       "      <td>0.164</td>\n",
       "      <td>150</td>\n",
       "      <td>0.311</td>\n",
       "      <td>150</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Now faith is confidence in what we hope for an...</td>\n",
       "      <td>0.391</td>\n",
       "      <td>150</td>\n",
       "      <td>0.410</td>\n",
       "      <td>150</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Take heed to yourselves. If your brother sins ...</td>\n",
       "      <td>-0.011</td>\n",
       "      <td>150</td>\n",
       "      <td>0.124</td>\n",
       "      <td>150</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Therefore, if anyone is in Christ, he is a new...</td>\n",
       "      <td>0.307</td>\n",
       "      <td>150</td>\n",
       "      <td>0.414</td>\n",
       "      <td>150</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>The Lord does not look at the things people lo...</td>\n",
       "      <td>0.432</td>\n",
       "      <td>150</td>\n",
       "      <td>0.508</td>\n",
       "      <td>150</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                lemma  krippendorff_agreement  \\\n",
       "34                                that all may be one                   0.118   \n",
       "37                                Seventy times seven                   0.494   \n",
       "19                               turn the other cheek                  -0.036   \n",
       "21                             Seek and you will find                   0.210   \n",
       "14                            The Lord is my shepherd                   0.213   \n",
       "11                        the truth will set you free                   0.250   \n",
       "3                       Love is patient, love is kind                   0.282   \n",
       "35                   For everything there is a season                   0.263   \n",
       "2                    Judge not, that ye be not judged                   0.472   \n",
       "28                  all things work together for good                   0.110   \n",
       "13                 For we walk by faith, not by sight                   0.383   \n",
       "40               The Lord will keep you from all harm                   0.169   \n",
       "36              Give to Caesar what belongs to Caesar                   0.196   \n",
       "39             Better is open rebuke than hidden love                   0.431   \n",
       "6              You shall have no other gods before me                   0.259   \n",
       "10            In the beginning God created the heaven                   0.151   \n",
       "16          If a man lies with a male as with a woman                   0.315   \n",
       "26          Love one another with brotherly affection                   0.410   \n",
       "23         Be strong and courageous. Do not be afraid                   0.223   \n",
       "33        Everything is possible for one who believes                   0.081   \n",
       "17  I can do all things through Christ who strengt...                   0.128   \n",
       "4   Husbands, love your wives, as Christ loved the...                   0.487   \n",
       "31  Love your enemies and pray for those who perse...                   0.073   \n",
       "18  My command is this: Love each other as I have ...                   0.288   \n",
       "41  you are precious in my eyes and honored, and I...                   0.421   \n",
       "32  The rain came down, the streams rose, and the ...                   0.166   \n",
       "20  Hatred stirs up conflict, but love covers over...                   0.172   \n",
       "1   But I suffer not a woman to teach, nor to usur...                   0.302   \n",
       "0   They have sown the wind, and they shall reap t...                   0.261   \n",
       "30  Anxiety weighs down the heart, but a kind word...                   0.093   \n",
       "12  Whoever does not love does not know God, becau...                   0.329   \n",
       "25  You are altogether beautiful, my darling; ther...                   0.385   \n",
       "15  You shall not lie with a male as with a woman;...                   0.423   \n",
       "24  This is the day that the Lord has made; let us...                   0.294   \n",
       "22  A wife of noble character who can find? She is...                   0.108   \n",
       "38  Greater love has no one than this: to lay down...                   0.347   \n",
       "27  Come to me, all you who labour and are overbur...                  -0.007   \n",
       "5   The heart is deceitful above all things, and d...                   0.164   \n",
       "8   Now faith is confidence in what we hope for an...                   0.391   \n",
       "29  Take heed to yourselves. If your brother sins ...                  -0.011   \n",
       "7   Therefore, if anyone is in Christ, he is a new...                   0.307   \n",
       "9   The Lord does not look at the things people lo...                   0.432   \n",
       "\n",
       "    n_instances_x  avg_pairwise_spearman_agreement  n_instances_y  length  \n",
       "34            150                            0.183            150      19  \n",
       "37            150                            0.619            150      19  \n",
       "19            150                            0.510            150      20  \n",
       "21            150                            0.558            150      22  \n",
       "14            150                            0.476            150      23  \n",
       "11            150                            0.347            150      27  \n",
       "3             150                            0.382            150      29  \n",
       "35            150                            0.443            150      32  \n",
       "2             150                            0.555            150      32  \n",
       "28            150                            0.543            150      33  \n",
       "13            150                            0.437            150      34  \n",
       "40            150                            0.213            150      36  \n",
       "36            150                            0.460            150      37  \n",
       "39            150                            0.481            150      38  \n",
       "6             150                            0.331            150      38  \n",
       "10            150                            0.219            150      39  \n",
       "16            150                            0.339            150      41  \n",
       "26            150                            0.571            150      41  \n",
       "23            150                            0.281            150      42  \n",
       "33            150                            0.118            150      43  \n",
       "17            150                            0.349            150      53  \n",
       "4             150                            0.507            150      53  \n",
       "31            150                            0.389            150      54  \n",
       "18            150                            0.426            150      55  \n",
       "41            150                            0.625            150      55  \n",
       "32            150                            0.406            150      56  \n",
       "20            150                            0.377            150      57  \n",
       "1             150                            0.315            150      57  \n",
       "0             150                            0.423            150      58  \n",
       "30            150                            0.253            150      59  \n",
       "12            150                            0.373            150      60  \n",
       "25            150                            0.537            150      65  \n",
       "15            150                            0.458            150      67  \n",
       "24            150                            0.492            150      72  \n",
       "22            150                            0.261            150      73  \n",
       "38            150                            0.640            150      75  \n",
       "27            150                            0.268            150      78  \n",
       "5             150                            0.311            150      81  \n",
       "8             150                            0.410            150      82  \n",
       "29            150                            0.124            150     101  \n",
       "7             150                            0.414            150     106  \n",
       "9             150                            0.508            150     123  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aps_targets = inter_annotator_agreement_per_target(df) # average pairwise spearman correlation\n",
    "ka_targets = krippendorff_agreement_per_target(df)\n",
    "aps_ka_targets = pd.merge(ka_targets, aps_targets, on='lemma')\n",
    "aps_ka_targets['length'] = aps_ka_targets['lemma'].apply(lambda x: len(x))\n",
    "aps_ka_targets.sort_values(by='length')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e9492f64-9e07-474a-826a-b04d92443dc0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.511, 0.423, '40 out of 42 targets')\n"
     ]
    }
   ],
   "source": [
    "targets = aps_ka_targets[(aps_ka_targets['avg_pairwise_spearman_agreement'] > 0.15)].lemma.values\n",
    "new_aps = inter_annotator_agreement(df, targets) # average pairwise spearman correlation\n",
    "new_ka = krippendorff_agreement(df, targets)\n",
    "print((new_aps, new_ka, f'{len(targets)} out of {df.lemma.unique().shape[0]} targets'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "309f27d7-2a28-4f12-923a-170892b5257c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.811, 0.709, '3821 out of 6300 instances')\n"
     ]
    }
   ],
   "source": [
    "# remove instances with judgment difference greater than 1\n",
    "tmp = df[['instanceID', 'label']].groupby('instanceID').agg(['unique']).reset_index()\n",
    "tmp[('label', 'unique')] = [max(i) - min(i) < 2 for i in tmp[('label', 'unique')]]\n",
    "filtered_instances = tmp[tmp[('label', 'unique')]].instanceID.values\n",
    "\n",
    "# remove instances with avg judgment between 2 and 3\n",
    "tmp = df[['instanceID', 'label']].groupby('instanceID').mean().reset_index()\n",
    "tmp['label'] = [0 if i <= 2 else i for i in tmp['label']]\n",
    "tmp['label'] = [1 if i >= 3 else i for i in tmp['label']]\n",
    "tmp = tmp[(tmp['label'].isin([0,1])) & (tmp['instanceID'].isin(filtered_instances))]\n",
    "#print(np.unique(tmp.label.values, return_counts=True))\n",
    "instances = tmp.instanceID.values\n",
    "\n",
    "new_aps = inter_annotator_agreement(df, instances=instances) # average pairwise spearman correlation\n",
    "new_ka = krippendorff_agreement(df, instances=instances)\n",
    "print((new_aps, new_ka, f'{len(instances)} out of {df.instanceID.unique().shape[0]} instances'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "a287cb78-1279-41e1-a4e4-6db57b060e0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lemma</th>\n",
       "      <th>krippendorff_agreement</th>\n",
       "      <th>n_instances_x</th>\n",
       "      <th>avg_pairwise_spearman_agreement</th>\n",
       "      <th>n_instances_y</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>that all may be one</td>\n",
       "      <td>0.677</td>\n",
       "      <td>79</td>\n",
       "      <td>0.782</td>\n",
       "      <td>79</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Seventy times seven</td>\n",
       "      <td>0.764</td>\n",
       "      <td>95</td>\n",
       "      <td>0.857</td>\n",
       "      <td>95</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>turn the other cheek</td>\n",
       "      <td>-0.193</td>\n",
       "      <td>120</td>\n",
       "      <td>0.405</td>\n",
       "      <td>120</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Seek and you will find</td>\n",
       "      <td>0.097</td>\n",
       "      <td>125</td>\n",
       "      <td>0.475</td>\n",
       "      <td>125</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>The Lord is my shepherd</td>\n",
       "      <td>0.138</td>\n",
       "      <td>117</td>\n",
       "      <td>0.431</td>\n",
       "      <td>117</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>the truth will set you free</td>\n",
       "      <td>0.217</td>\n",
       "      <td>101</td>\n",
       "      <td>0.368</td>\n",
       "      <td>101</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Love is patient, love is kind</td>\n",
       "      <td>0.798</td>\n",
       "      <td>61</td>\n",
       "      <td>0.834</td>\n",
       "      <td>61</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>For everything there is a season</td>\n",
       "      <td>0.369</td>\n",
       "      <td>104</td>\n",
       "      <td>0.557</td>\n",
       "      <td>104</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Judge not, that ye be not judged</td>\n",
       "      <td>0.450</td>\n",
       "      <td>96</td>\n",
       "      <td>0.469</td>\n",
       "      <td>96</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>all things work together for good</td>\n",
       "      <td>-0.030</td>\n",
       "      <td>124</td>\n",
       "      <td>0.430</td>\n",
       "      <td>124</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>For we walk by faith, not by sight</td>\n",
       "      <td>0.823</td>\n",
       "      <td>79</td>\n",
       "      <td>0.866</td>\n",
       "      <td>79</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>The Lord will keep you from all harm</td>\n",
       "      <td>0.178</td>\n",
       "      <td>103</td>\n",
       "      <td>0.166</td>\n",
       "      <td>103</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Give to Caesar what belongs to Caesar</td>\n",
       "      <td>0.117</td>\n",
       "      <td>106</td>\n",
       "      <td>0.522</td>\n",
       "      <td>106</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Better is open rebuke than hidden love</td>\n",
       "      <td>0.828</td>\n",
       "      <td>87</td>\n",
       "      <td>0.911</td>\n",
       "      <td>87</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>You shall have no other gods before me</td>\n",
       "      <td>0.506</td>\n",
       "      <td>80</td>\n",
       "      <td>0.646</td>\n",
       "      <td>80</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>In the beginning God created the heaven</td>\n",
       "      <td>0.177</td>\n",
       "      <td>66</td>\n",
       "      <td>0.277</td>\n",
       "      <td>66</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>If a man lies with a male as with a woman</td>\n",
       "      <td>0.492</td>\n",
       "      <td>86</td>\n",
       "      <td>0.517</td>\n",
       "      <td>86</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Love one another with brotherly affection</td>\n",
       "      <td>0.566</td>\n",
       "      <td>101</td>\n",
       "      <td>0.657</td>\n",
       "      <td>101</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Be strong and courageous. Do not be afraid</td>\n",
       "      <td>0.822</td>\n",
       "      <td>69</td>\n",
       "      <td>0.833</td>\n",
       "      <td>69</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Everything is possible for one who believes</td>\n",
       "      <td>0.509</td>\n",
       "      <td>81</td>\n",
       "      <td>0.557</td>\n",
       "      <td>81</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>I can do all things through Christ who strengt...</td>\n",
       "      <td>0.624</td>\n",
       "      <td>73</td>\n",
       "      <td>0.787</td>\n",
       "      <td>73</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Husbands, love your wives, as Christ loved the...</td>\n",
       "      <td>0.802</td>\n",
       "      <td>95</td>\n",
       "      <td>0.802</td>\n",
       "      <td>95</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Love your enemies and pray for those who perse...</td>\n",
       "      <td>-0.004</td>\n",
       "      <td>104</td>\n",
       "      <td>0.532</td>\n",
       "      <td>104</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>My command is this: Love each other as I have ...</td>\n",
       "      <td>0.589</td>\n",
       "      <td>97</td>\n",
       "      <td>0.790</td>\n",
       "      <td>97</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>you are precious in my eyes and honored, and I...</td>\n",
       "      <td>0.439</td>\n",
       "      <td>110</td>\n",
       "      <td>0.730</td>\n",
       "      <td>110</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>The rain came down, the streams rose, and the ...</td>\n",
       "      <td>0.625</td>\n",
       "      <td>71</td>\n",
       "      <td>0.802</td>\n",
       "      <td>71</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Hatred stirs up conflict, but love covers over...</td>\n",
       "      <td>0.148</td>\n",
       "      <td>100</td>\n",
       "      <td>0.517</td>\n",
       "      <td>100</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>But I suffer not a woman to teach, nor to usur...</td>\n",
       "      <td>0.232</td>\n",
       "      <td>71</td>\n",
       "      <td>0.217</td>\n",
       "      <td>71</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>They have sown the wind, and they shall reap t...</td>\n",
       "      <td>0.621</td>\n",
       "      <td>55</td>\n",
       "      <td>0.668</td>\n",
       "      <td>55</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Anxiety weighs down the heart, but a kind word...</td>\n",
       "      <td>0.518</td>\n",
       "      <td>81</td>\n",
       "      <td>0.777</td>\n",
       "      <td>81</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Whoever does not love does not know God, becau...</td>\n",
       "      <td>0.355</td>\n",
       "      <td>121</td>\n",
       "      <td>0.393</td>\n",
       "      <td>121</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>You are altogether beautiful, my darling; ther...</td>\n",
       "      <td>0.782</td>\n",
       "      <td>92</td>\n",
       "      <td>0.878</td>\n",
       "      <td>92</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>You shall not lie with a male as with a woman;...</td>\n",
       "      <td>0.648</td>\n",
       "      <td>101</td>\n",
       "      <td>0.753</td>\n",
       "      <td>101</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>This is the day that the Lord has made; let us...</td>\n",
       "      <td>0.847</td>\n",
       "      <td>77</td>\n",
       "      <td>0.884</td>\n",
       "      <td>77</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>A wife of noble character who can find? She is...</td>\n",
       "      <td>0.775</td>\n",
       "      <td>74</td>\n",
       "      <td>0.797</td>\n",
       "      <td>74</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Greater love has no one than this: to lay down...</td>\n",
       "      <td>0.355</td>\n",
       "      <td>125</td>\n",
       "      <td>0.737</td>\n",
       "      <td>125</td>\n",
       "      <td>75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Come to me, all you who labour and are overbur...</td>\n",
       "      <td>-0.024</td>\n",
       "      <td>101</td>\n",
       "      <td>0.355</td>\n",
       "      <td>101</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>The heart is deceitful above all things, and d...</td>\n",
       "      <td>0.432</td>\n",
       "      <td>75</td>\n",
       "      <td>0.591</td>\n",
       "      <td>75</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Now faith is confidence in what we hope for an...</td>\n",
       "      <td>0.853</td>\n",
       "      <td>72</td>\n",
       "      <td>0.870</td>\n",
       "      <td>72</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Take heed to yourselves. If your brother sins ...</td>\n",
       "      <td>0.267</td>\n",
       "      <td>91</td>\n",
       "      <td>0.485</td>\n",
       "      <td>91</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Therefore, if anyone is in Christ, he is a new...</td>\n",
       "      <td>0.655</td>\n",
       "      <td>75</td>\n",
       "      <td>0.744</td>\n",
       "      <td>75</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>The Lord does not look at the things people lo...</td>\n",
       "      <td>0.665</td>\n",
       "      <td>80</td>\n",
       "      <td>0.733</td>\n",
       "      <td>80</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                lemma  krippendorff_agreement  \\\n",
       "34                                that all may be one                   0.677   \n",
       "37                                Seventy times seven                   0.764   \n",
       "19                               turn the other cheek                  -0.193   \n",
       "21                             Seek and you will find                   0.097   \n",
       "14                            The Lord is my shepherd                   0.138   \n",
       "11                        the truth will set you free                   0.217   \n",
       "3                       Love is patient, love is kind                   0.798   \n",
       "35                   For everything there is a season                   0.369   \n",
       "2                    Judge not, that ye be not judged                   0.450   \n",
       "28                  all things work together for good                  -0.030   \n",
       "13                 For we walk by faith, not by sight                   0.823   \n",
       "40               The Lord will keep you from all harm                   0.178   \n",
       "36              Give to Caesar what belongs to Caesar                   0.117   \n",
       "39             Better is open rebuke than hidden love                   0.828   \n",
       "6              You shall have no other gods before me                   0.506   \n",
       "10            In the beginning God created the heaven                   0.177   \n",
       "16          If a man lies with a male as with a woman                   0.492   \n",
       "26          Love one another with brotherly affection                   0.566   \n",
       "23         Be strong and courageous. Do not be afraid                   0.822   \n",
       "33        Everything is possible for one who believes                   0.509   \n",
       "17  I can do all things through Christ who strengt...                   0.624   \n",
       "4   Husbands, love your wives, as Christ loved the...                   0.802   \n",
       "31  Love your enemies and pray for those who perse...                  -0.004   \n",
       "18  My command is this: Love each other as I have ...                   0.589   \n",
       "41  you are precious in my eyes and honored, and I...                   0.439   \n",
       "32  The rain came down, the streams rose, and the ...                   0.625   \n",
       "20  Hatred stirs up conflict, but love covers over...                   0.148   \n",
       "1   But I suffer not a woman to teach, nor to usur...                   0.232   \n",
       "0   They have sown the wind, and they shall reap t...                   0.621   \n",
       "30  Anxiety weighs down the heart, but a kind word...                   0.518   \n",
       "12  Whoever does not love does not know God, becau...                   0.355   \n",
       "25  You are altogether beautiful, my darling; ther...                   0.782   \n",
       "15  You shall not lie with a male as with a woman;...                   0.648   \n",
       "24  This is the day that the Lord has made; let us...                   0.847   \n",
       "22  A wife of noble character who can find? She is...                   0.775   \n",
       "38  Greater love has no one than this: to lay down...                   0.355   \n",
       "27  Come to me, all you who labour and are overbur...                  -0.024   \n",
       "5   The heart is deceitful above all things, and d...                   0.432   \n",
       "8   Now faith is confidence in what we hope for an...                   0.853   \n",
       "29  Take heed to yourselves. If your brother sins ...                   0.267   \n",
       "7   Therefore, if anyone is in Christ, he is a new...                   0.655   \n",
       "9   The Lord does not look at the things people lo...                   0.665   \n",
       "\n",
       "    n_instances_x  avg_pairwise_spearman_agreement  n_instances_y  length  \n",
       "34             79                            0.782             79      19  \n",
       "37             95                            0.857             95      19  \n",
       "19            120                            0.405            120      20  \n",
       "21            125                            0.475            125      22  \n",
       "14            117                            0.431            117      23  \n",
       "11            101                            0.368            101      27  \n",
       "3              61                            0.834             61      29  \n",
       "35            104                            0.557            104      32  \n",
       "2              96                            0.469             96      32  \n",
       "28            124                            0.430            124      33  \n",
       "13             79                            0.866             79      34  \n",
       "40            103                            0.166            103      36  \n",
       "36            106                            0.522            106      37  \n",
       "39             87                            0.911             87      38  \n",
       "6              80                            0.646             80      38  \n",
       "10             66                            0.277             66      39  \n",
       "16             86                            0.517             86      41  \n",
       "26            101                            0.657            101      41  \n",
       "23             69                            0.833             69      42  \n",
       "33             81                            0.557             81      43  \n",
       "17             73                            0.787             73      53  \n",
       "4              95                            0.802             95      53  \n",
       "31            104                            0.532            104      54  \n",
       "18             97                            0.790             97      55  \n",
       "41            110                            0.730            110      55  \n",
       "32             71                            0.802             71      56  \n",
       "20            100                            0.517            100      57  \n",
       "1              71                            0.217             71      57  \n",
       "0              55                            0.668             55      58  \n",
       "30             81                            0.777             81      59  \n",
       "12            121                            0.393            121      60  \n",
       "25             92                            0.878             92      65  \n",
       "15            101                            0.753            101      67  \n",
       "24             77                            0.884             77      72  \n",
       "22             74                            0.797             74      73  \n",
       "38            125                            0.737            125      75  \n",
       "27            101                            0.355            101      78  \n",
       "5              75                            0.591             75      81  \n",
       "8              72                            0.870             72      82  \n",
       "29             91                            0.485             91     101  \n",
       "7              75                            0.744             75     106  \n",
       "9              80                            0.733             80     123  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aps_targets = inter_annotator_agreement_per_target(df, instances=instances) # average pairwise spearman correlation\n",
    "ka_targets = krippendorff_agreement_per_target(df, instances=instances)\n",
    "aps_ka_targets = pd.merge(ka_targets, aps_targets, on='lemma')\n",
    "aps_ka_targets['length'] = aps_ka_targets['lemma'].apply(lambda x: len(x))\n",
    "aps_ka_targets.sort_values(by='length')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cf0495e-a298-4a1d-9576-a67d7ca6fdfe",
   "metadata": {},
   "source": [
    "# DURel format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7f0ec6f9-864d-4201-b712-af06f028767d",
   "metadata": {},
   "outputs": [],
   "source": [
    "durel_uses = df_uses.copy()\n",
    "durel_judgments = df.copy()\n",
    "\n",
    "# DURel preprocessing\n",
    "durel_judgments['lemma'] = durel_judgments['lemma'].apply(lambda x: x.replace(' ', '_').replace(',', ''))\n",
    "durel_judgments['timestamp'] = ''\n",
    "durel_judgments = durel_judgments.rename(columns={'dataID1': 'identifier1', \n",
    "                                                  'dataID2': 'identifier2', \n",
    "                                                  'label':'judgment'})\n",
    "columns = ['identifier1', 'identifier2', 'annotator', 'judgment', 'comment', 'lemma', 'timestamp']\n",
    "durel_judgments = durel_judgments[columns]\n",
    "durel_judgments['judgment'] = [j if j == j else 0.0 for j in durel_judgments['judgment']]\n",
    "\n",
    "durel_uses['lemma'] = durel_uses['lemma'].apply(lambda x: x.replace(' ', '_').replace(',', ''))\n",
    "durel_uses['description'] = \"\"\n",
    "durel_uses['pos'] = \"\"\n",
    "durel_uses['date'] = \"\"\n",
    "durel_uses['grouping'] = \"\"\n",
    "durel_uses = durel_uses.rename(columns={'dataID': 'identifier', \n",
    "                                        'indices_target_sentence': 'indexes_target_sentence', \n",
    "                                        'indices_target_token': 'indexes_target_token'})\n",
    "columns = ['lemma', 'pos', 'date', 'grouping', 'identifier', 'description', 'context', 'indexes_target_token', 'indexes_target_sentence']\n",
    "durel_uses = durel_uses[columns]\n",
    "\n",
    "targets = durel_uses.lemma.unique()\n",
    "for target in targets:    \n",
    "    df_target_uses = durel_uses[durel_uses['lemma'] == target]\n",
    "    df_target_judgments = durel_judgments[durel_judgments['lemma'] == target]\n",
    "\n",
    "    # find id quote\n",
    "    id_quote = re.search('\\(.*\\)', df_target_uses.iloc[0].identifier).group(0).replace(':', ' ')\n",
    "\n",
    "    # make dirs and store dataframe\n",
    "    Path(f'TRoTR/DURel_data/{id_quote}').mkdir(parents=True, exist_ok=True)\n",
    "    Path(f'TRoTR/DURel_data/uses').mkdir(parents=True, exist_ok=True)\n",
    "    df_target_uses.to_csv(f'TRoTR/DURel_data/{id_quote}/uses.tsv', index=False, sep='\\t', quoting=csv.QUOTE_NONE)\n",
    "    df_target_uses.to_csv(f'TRoTR/DURel_data/uses/{id_quote}.tsv', index=False, sep='\\t', quoting=csv.QUOTE_NONE)\n",
    "\n",
    "    # make dirs and store dataframe\n",
    "    Path(f'TRoTR/DURel_data/judgments').mkdir(parents=True, exist_ok=True)\n",
    "    df_target_judgments.to_csv(f'TRoTR/DURel_data/{id_quote}/judgments.tsv', index=False, sep='\\t', quoting=csv.QUOTE_NONE)\n",
    "    df_target_judgments.to_csv(f'TRoTR/DURel_data/judgments/{id_quote}.tsv', index=False, sep='\\t', quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0eb026fa-19b4-4650-8600-bead58b501a9",
   "metadata": {},
   "source": [
    "Number of pairs with more than one 'cannot_decide'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4913f1d-c536-42c9-a409-1c4826f61962",
   "metadata": {},
   "outputs": [],
   "source": [
    "cannot_decide = df[~df['label'].isin([1,2,3,4])].fillna('-').groupby(['instanceID', 'label']).count().reset_index()\n",
    "cannot_decide[cannot_decide['annotator'] > 1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}