05_Experiment_01_Preliminary_Analysis_Annotation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5/14 (Tue) | Experiment\n",
    "\n",
    "# Preliminary Analyses of Annoation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Introduction\n",
    "\n",
    "This notebook conducts preliminary analyses.\n",
    "The goal of current analyses is to fill the following table.\n",
    "\n",
    "| Task | WER | N disfluency (Manual / Automatic) | N MCP (Manual / Automatic) | N ECP (Manual / Automatic) |\n",
    "| - | - | - | - | - |\n",
    "| Arg_Oly |  |  |  |  |\n",
    "| Cartoon |  |  |  |  |\n",
    "| RtSwithoutRAA |  |  |  |  |\n",
    "| RtSwithRAA |  |  |  |  |\n",
    "| Monologue |  |  |  |  |\n",
    "| WoZ_Interview |  |  |  |  |\n",
    "| ALL |  |  |  |  |\n",
    "\n",
    "Before starting the analyses, the following code block loads required packages and define global variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List, Tuple, Dict, Generator, Optional\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from jiwer import wer\n",
    "\n",
    "from utils.mfr import logit_2_rating\n",
    "\n",
    "DATA_DIR = Path(\"/home/matsuura/Development/app/feature_extraction_api/experiment/data\")\n",
    "\n",
    "MONOLOGUE_TASK = [\"Arg_Oly\", \"Cartoon\", \"RtSwithoutRAA\", \"RtSwithRAA\"]\n",
    "DIALOGUE_TASK = [\"WoZ_Interview\"]\n",
    "\n",
    "FILLER = {\"uh\", \"ah\", \"um\", \"mm\", \"hmm\", \"oh\", \"mm-hmm\", \"er\", \"mhm\", \"uh-huh\", \"er\", \"erm\", \"huh\", \"uhu\", \"mmhmm\", \"uhhuh\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## 2. Define Functions\n",
    "\n",
    "This section defines functions for the preliminary analyses.\n",
    "The following code block defines two functions; one generates csv file paths of manual and automatic annotation results; and another one loads them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def annotation_result_csv_path_generator(task: str, rating_filter: Optional[List[int]] =None) -> Generator[Tuple[Path, Path], None, None]:\n",
    "    load_dir = DATA_DIR / f\"{task}/10_SCTK_Inputs\"\n",
    "\n",
    "    if rating_filter is None:\n",
    "        for manu_csv_path in load_dir.glob(\"*_manu.csv\"):\n",
    "            filename = manu_csv_path.stem.removesuffix(\"_manu\")\n",
    "            auto_csv_path = load_dir / f\"{filename}_auto.csv\"\n",
    "\n",
    "            yield manu_csv_path, auto_csv_path\n",
    "    else:\n",
    "        pf_path = DATA_DIR / f\"{task}/12_PF_Rating/pf_rating.csv\"\n",
    "        df_pf = pd.read_csv(pf_path)\n",
    "        uid_list = df_pf[\"uid\"].to_numpy()\n",
    "\n",
    "        logit_path = pf_path.parent / \"logit.csv\"\n",
    "        threshold_path = logit_path.parent / \"threshold.csv\"\n",
    "        \n",
    "        df_logit = pd.read_csv(logit_path, index_col=0)\n",
    "        rating_list = logit_2_rating(df_logit[\"theta\"], threshold_path)\n",
    "\n",
    "        mask = np.full(rating_list.shape, False, dtype=bool)\n",
    "        for rating in rating_filter:\n",
    "            mask = mask | (rating_list == rating)\n",
    "        \n",
    "        uid_list = uid_list[mask]\n",
    "\n",
    "        for uid in uid_list:\n",
    "            if task == \"WoZ_Interview\":\n",
    "                uid = str(int(uid)).zfill(3)\n",
    "\n",
    "            filename_pattern = f\"{uid}*_manu.csv\"\n",
    "            for manu_csv_path in load_dir.glob(filename_pattern):\n",
    "                filename = manu_csv_path.stem.removesuffix(\"_manu\")\n",
    "                auto_csv_path = load_dir / f\"{filename}_auto.csv\"\n",
    "\n",
    "                yield manu_csv_path, auto_csv_path\n",
    "\n",
    "def load_dataset(\n",
    "        rating_filter_monologue: Optional[List[int]] =None,\n",
    "        rating_filter_dialogue: Optional[List[int]] =None,\n",
    ") -> Dict[str, Dict[str, List[Dict[str, pd.DataFrame]]]]:\n",
    "    dataset = {\n",
    "        \"monologue\": {},\n",
    "        \"dialogue\": {}\n",
    "    }\n",
    "    \n",
    "    for monologue_task in MONOLOGUE_TASK:\n",
    "        dataset[\"monologue\"][monologue_task] = []\n",
    "        \n",
    "        for manu_csv_path, auto_csv_path in annotation_result_csv_path_generator(monologue_task, rating_filter=rating_filter_monologue):\n",
    "            df_manu = pd.read_csv(manu_csv_path)\n",
    "            df_auto = pd.DataFrame([], columns=[\"text\"])\n",
    "            if auto_csv_path.exists():\n",
    "                df_auto = pd.read_csv(auto_csv_path)\n",
    "\n",
    "            dataset[\"monologue\"][monologue_task].append({\n",
    "                \"manual\": df_manu,\n",
    "                \"automatic\": df_auto\n",
    "            })\n",
    "\n",
    "    for dialogue_task in DIALOGUE_TASK:\n",
    "        dataset[\"dialogue\"][dialogue_task] = []\n",
    "\n",
    "        for manu_csv_path, auto_csv_path in annotation_result_csv_path_generator(dialogue_task, rating_filter=rating_filter_dialogue):\n",
    "            df_manu = pd.read_csv(manu_csv_path, na_values=[\"\", \" \"], keep_default_na=False)\n",
    "            df_auto = pd.DataFrame([], columns=[\"text\"])\n",
    "            if auto_csv_path.exists():\n",
    "                df_auto = pd.read_csv(auto_csv_path, na_values=[\"\", \" \"], keep_default_na=False)\n",
    "\n",
    "            dataset[\"dialogue\"][dialogue_task].append({\n",
    "                \"manual\": df_manu,\n",
    "                \"automatic\": df_auto\n",
    "            })\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block defines a function to calculate WER."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_wer(annotation_results: List[Dict[str, pd.DataFrame]], remove_filer: bool =False) -> float:\n",
    "    ref = []\n",
    "    hyp = []\n",
    "\n",
    "    for annotation_result in annotation_results:\n",
    "        df_manu = annotation_result[\"manual\"]\n",
    "        df_auto = annotation_result[\"automatic\"]\n",
    "\n",
    "        mask_tag_manu = df_manu[\"text\"].astype(str).str.endswith(\">\")\n",
    "        mask_tag_auto = df_auto[\"text\"].astype(str).str.endswith(\">\")\n",
    "\n",
    "        df_manu = df_manu[~mask_tag_manu]\n",
    "        df_auto = df_auto[~mask_tag_auto]\n",
    "\n",
    "        if remove_filer:\n",
    "            for filler in FILLER:\n",
    "                mask_filler_manu = (df_manu[\"text\"] == filler)\n",
    "                df_manu = df_manu[~mask_filler_manu]\n",
    "\n",
    "                mask_filler_auto = (df_auto[\"text\"] == filler)\n",
    "                df_auto = df_auto[~mask_filler_auto]\n",
    "\n",
    "        text_manu = \" \".join(df_manu[\"text\"].astype(str))\n",
    "        text_auto = \" \".join(df_auto[\"text\"].astype(str))\n",
    "\n",
    "        if len(text_manu) == 0 or len(text_auto) == 0:\n",
    "            continue\n",
    "\n",
    "        ref.append(text_manu)\n",
    "        hyp.append(text_auto)\n",
    "\n",
    "    return wer(ref, hyp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block defines a function to count the number of tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_tags(annotation_results: List[Dict[str, pd.DataFrame]], target_tag: str) -> Tuple[List[int], List[int]]:\n",
    "    n_tag_manu = []\n",
    "    n_tag_auto = []\n",
    "\n",
    "    for annotation_result in annotation_results:\n",
    "        df_manu = annotation_result[\"manual\"]\n",
    "        df_auto = annotation_result[\"automatic\"]\n",
    "\n",
    "        mask_tag_manu = (df_manu[\"text\"] == target_tag)\n",
    "        mask_tag_auto = (df_auto[\"text\"] == target_tag)\n",
    "\n",
    "        n_tag_manu.append(mask_tag_manu.sum())\n",
    "        n_tag_auto.append(mask_tag_auto.sum())\n",
    "\n",
    "    return n_tag_manu, n_tag_auto"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## 3. Preliminary Analyses\n",
    "\n",
    "This section conducts the preliminary analyses.\n",
    "The following code block loads entire dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>type</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.220000</td>\n",
       "      <td>0.240063</td>\n",
       "      <td>01_text</td>\n",
       "      <td>i</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.300063</td>\n",
       "      <td>0.660125</td>\n",
       "      <td>01_text</td>\n",
       "      <td>agree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.820187</td>\n",
       "      <td>1.040250</td>\n",
       "      <td>01_text</td>\n",
       "      <td>this</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.260312</td>\n",
       "      <td>2.080563</td>\n",
       "      <td>01_text</td>\n",
       "      <td>statement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.132178</td>\n",
       "      <td>2.764380</td>\n",
       "      <td>02_pause</td>\n",
       "      <td>&lt;CE&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   start_time  end_time      type       text\n",
       "0    0.220000  0.240063   01_text          i\n",
       "1    0.300063  0.660125   01_text      agree\n",
       "2    0.820187  1.040250   01_text       this\n",
       "3    1.260312  2.080563   01_text  statement\n",
       "4    2.132178  2.764380  02_pause       <CE>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[\"monologue\"][\"Arg_Oly\"][0][\"manual\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>type</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.220000</td>\n",
       "      <td>0.240063</td>\n",
       "      <td>01_text</td>\n",
       "      <td>i</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.300063</td>\n",
       "      <td>0.660125</td>\n",
       "      <td>01_text</td>\n",
       "      <td>agree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.820187</td>\n",
       "      <td>1.040250</td>\n",
       "      <td>01_text</td>\n",
       "      <td>this</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.260312</td>\n",
       "      <td>2.080563</td>\n",
       "      <td>01_text</td>\n",
       "      <td>statement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.080563</td>\n",
       "      <td>2.740750</td>\n",
       "      <td>02_pause</td>\n",
       "      <td>&lt;CI&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   start_time  end_time      type       text\n",
       "0    0.220000  0.240063   01_text          i\n",
       "1    0.300063  0.660125   01_text      agree\n",
       "2    0.820187  1.040250   01_text       this\n",
       "3    1.260312  2.080563   01_text  statement\n",
       "4    2.080563  2.740750  02_pause       <CI>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[\"monologue\"][\"Arg_Oly\"][0][\"automatic\"].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1. WER\n",
    "\n",
    "The following code block calculate WER of monologue tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of Arg_Oly = 0.1602270395951583\n",
      "WER of Cartoon = 0.13862017646433508\n",
      "WER of RtSwithoutRAA = 0.1987206157686707\n",
      "WER of RtSwithRAA = 0.20201547971533945\n",
      "WER of monologue task = 0.17605261694198787\n"
     ]
    }
   ],
   "source": [
    "monologue_data = []\n",
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {monologue_task} = {res}\")\n",
    "\n",
    "    monologue_data += annotation_results\n",
    "\n",
    "res = calculate_wer(monologue_data, remove_filer=True)\n",
    "print(f\"WER of monologue task = {res}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculate WER of a dialogue task."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of WoZ_Interview = 0.15021711724331138\n"
     ]
    }
   ],
   "source": [
    "dialogue_data = []\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {dialogue_task} = {res}\")\n",
    "\n",
    "    dialogue_data += annotation_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calcualte WER of the entire tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of all tasks = 0.1674828781444276\n"
     ]
    }
   ],
   "source": [
    "all_task_data = monologue_data + dialogue_data\n",
    "\n",
    "res = calculate_wer(all_task_data, remove_filer=True)\n",
    "print(f\"WER of all tasks = {res}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2. Count Disfluency\n",
    "\n",
    "The following code block counts the number of disfluency words in monologue tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_disfluency of Arg_Oly = 1876\n",
      "[Automatic] N_disfluency of Arg_Oly = 1793\n",
      "[Manual] N_disfluency of Cartoon = 2813\n",
      "[Automatic] N_disfluency of Cartoon = 2517\n",
      "[Manual] N_disfluency of RtSwithoutRAA = 2949\n",
      "[Automatic] N_disfluency of RtSwithoutRAA = 2542\n",
      "[Manual] N_disfluency of RtSwithRAA = 2887\n",
      "[Automatic] N_disfluency of RtSwithRAA = 2556\n",
      "[Manual] N_disfluency of monologue task = 10525\n",
      "[Automatic] N_disfluency of monologue task = 9408\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<DISFLUENCY>\")\n",
    "\n",
    "    print(f\"[Manual] N_disfluency of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_disfluency of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<DISFLUENCY>\")\n",
    "print(f\"[Manual] N_disfluency of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_disfluency of monologue task = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts the number of disfluency words in dialogue tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_disfluency of WoZ_Interview = 3935\n",
      "[Automatic] N_disfluency of WoZ_Interview = 3514\n"
     ]
    }
   ],
   "source": [
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<DISFLUENCY>\")\n",
    "\n",
    "    print(f\"[Manual] N_disfluency of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_disfluency of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts the number of disfluency words in the whole tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_disfluency of all tasks = 14460\n",
      "[Automatic] N_disfluency of all tasks = 12922\n"
     ]
    }
   ],
   "source": [
    "n_tags_manu, n_tags_auto = count_tags(all_task_data, \"<DISFLUENCY>\")\n",
    "print(f\"[Manual] N_disfluency of all tasks = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_disfluency of all tasks = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3. Count Mid-Clause Pauses\n",
    "\n",
    "The following code block counts the number of MCP in monologue tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_MCP of Arg_Oly = 4302\n",
      "[Automatic] N_MCP of Arg_Oly = 4923\n",
      "[Manual] N_MCP of Cartoon = 5559\n",
      "[Automatic] N_MCP of Cartoon = 6739\n",
      "[Manual] N_MCP of RtSwithoutRAA = 6169\n",
      "[Automatic] N_MCP of RtSwithoutRAA = 7367\n",
      "[Manual] N_MCP of RtSwithRAA = 6332\n",
      "[Automatic] N_MCP of RtSwithRAA = 7502\n",
      "[Manual] N_MCP of monologue task = 22362\n",
      "[Automatic] N_MCP of monologue task = 26531\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CI>\")\n",
    "\n",
    "    print(f\"[Manual] N_MCP of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_MCP of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<CI>\")\n",
    "print(f\"[Manual] N_MCP of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_MCP of monologue task = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts the number of MCP in dialogue tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_MCP of WoZ_Interview = 7574\n",
      "[Automatic] N_MCP of WoZ_Interview = 10322\n"
     ]
    }
   ],
   "source": [
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CI>\")\n",
    "\n",
    "    print(f\"[Manual] N_MCP of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_MCP of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts the number of disfluency words in the whole tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_MCP of all tasks = 29936\n",
      "[Automatic] N_MCP of all tasks = 36853\n"
     ]
    }
   ],
   "source": [
    "n_tags_manu, n_tags_auto = count_tags(all_task_data, \"<CI>\")\n",
    "print(f\"[Manual] N_MCP of all tasks = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_MCP of all tasks = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.4. Count End-Clause Pauses\n",
    "\n",
    "The following code block counts the number of ECP in monologue tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_ECP of Arg_Oly = 1142\n",
      "[Automatic] N_ECP of Arg_Oly = 1001\n",
      "[Manual] N_ECP of Cartoon = 1985\n",
      "[Automatic] N_ECP of Cartoon = 1810\n",
      "[Manual] N_ECP of RtSwithoutRAA = 1835\n",
      "[Automatic] N_ECP of RtSwithoutRAA = 1627\n",
      "[Manual] N_ECP of RtSwithRAA = 1842\n",
      "[Automatic] N_ECP of RtSwithRAA = 1573\n",
      "[Manual] N_ECP of monologue task = 6804\n",
      "[Automatic] N_ECP of monologue task = 6011\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CE>\")\n",
    "\n",
    "    print(f\"[Manual] N_ECP of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_ECP of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<CE>\")\n",
    "print(f\"[Manual] N_ECP of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_ECP of monologue task = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts the number of ECP in dialogue tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_ECP of WoZ_Interview = 2414\n",
      "[Automatic] N_ECP of WoZ_Interview = 2646\n"
     ]
    }
   ],
   "source": [
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CE>\")\n",
    "\n",
    "    print(f\"[Manual] N_ECP of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_ECP of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts the number of disfluency words in the whole tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_ECP of all tasks = 9218\n",
      "[Automatic] N_ECP of all tasks = 8657\n"
     ]
    }
   ],
   "source": [
    "n_tags_manu, n_tags_auto = count_tags(all_task_data, \"<CE>\")\n",
    "print(f\"[Manual] N_ECP of all tasks = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_ECP of all tasks = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## 4. Summary\n",
    "\n",
    "As results, the following table was obtained.\n",
    "\n",
    "| Task | WER | N disfluency (Manual / Automatic) | N MCP (Manual / Automatic) | N ECP (Manual / Automatic) |\n",
    "| - | - | - | - | - |\n",
    "| Arg_Oly | 16.1% | 1,876 / 1,793 | 4,302 / 4,923 | 1,142 / 1,001 |\n",
    "| Cartoon | 13.9% | 2,813 / 2,517 | 5,559 / 6,739 | 1,985 / 1,810 |\n",
    "| RtSwithoutRAA | 19.9% | 2,949 / 2,542 | 6,169 / 7,367 | 1,835 / 1,627 |\n",
    "| RtSwithRAA | 20.2% | 2,887 / 2,556 | 6,332 / 7,502 | 1,842 / 1,573 |\n",
    "| Monologue | 17.6% | 10,525 / 9,408 | 22,362 / 26,531 | 6,804 / 6,011 |\n",
    "| WoZ_Interview | 15.0% | 3,935 / 3,514 | 7,574 / 10,322 | 2,414 / 2,646 |\n",
    "| ALL | 16.8% | 14,460 / 12,922 | 29,936 / 36,853 | 9,218 / 8,657 |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## 5. Additional Analyses\n",
    "\n",
    "This section conducts the same analyses for each PF groups.\n",
    "\n",
    "### 5.1. Beginners\n",
    "\n",
    "The following code block loads beginners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "beginner_dataset = load_dataset(rating_filter_monologue=[0, 1, 2], rating_filter_dialogue=[0, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculates WER of beginners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of Arg_Oly = 0.20163060520539355\n",
      "WER of Cartoon = 0.2280612244897959\n",
      "WER of RtSwithoutRAA = 0.24742773150416464\n",
      "WER of RtSwithRAA = 0.27688442211055275\n",
      "WER of monologue task = 0.23625345334640407\n",
      "WER of WoZ_Interview = 0.2485251852972319\n",
      "WER of all tasks = 0.2408030506953791\n"
     ]
    }
   ],
   "source": [
    "monologue_data = []\n",
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {monologue_task} = {res}\")\n",
    "\n",
    "    monologue_data += annotation_results\n",
    "\n",
    "res = calculate_wer(monologue_data, remove_filer=True)\n",
    "print(f\"WER of monologue task = {res}\")\n",
    "\n",
    "dialogue_data = []\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {dialogue_task} = {res}\")\n",
    "\n",
    "    dialogue_data += annotation_results\n",
    "\n",
    "all_task_data = monologue_data + dialogue_data\n",
    "\n",
    "res = calculate_wer(all_task_data, remove_filer=True)\n",
    "print(f\"WER of all tasks = {res}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts disfluency tags in beginners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_disfluency of Arg_Oly = 609\n",
      "[Automatic] N_disfluency of Arg_Oly = 552\n",
      "[Manual] N_disfluency of Cartoon = 386\n",
      "[Automatic] N_disfluency of Cartoon = 351\n",
      "[Manual] N_disfluency of RtSwithoutRAA = 817\n",
      "[Automatic] N_disfluency of RtSwithoutRAA = 708\n",
      "[Manual] N_disfluency of RtSwithRAA = 374\n",
      "[Automatic] N_disfluency of RtSwithRAA = 369\n",
      "[Manual] N_disfluency of monologue task = 2186\n",
      "[Automatic] N_disfluency of monologue task = 1980\n",
      "[Manual] N_disfluency of WoZ_Interview = 955\n",
      "[Automatic] N_disfluency of WoZ_Interview = 730\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<DISFLUENCY>\")\n",
    "\n",
    "    print(f\"[Manual] N_disfluency of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_disfluency of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<DISFLUENCY>\")\n",
    "print(f\"[Manual] N_disfluency of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_disfluency of monologue task = {sum(n_tags_auto)}\")\n",
    "\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<DISFLUENCY>\")\n",
    "\n",
    "    print(f\"[Manual] N_disfluency of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_disfluency of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts MCP tags in beginners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_MCP of Arg_Oly = 1256\n",
      "[Automatic] N_MCP of Arg_Oly = 1283\n",
      "[Manual] N_MCP of Cartoon = 812\n",
      "[Automatic] N_MCP of Cartoon = 836\n",
      "[Manual] N_MCP of RtSwithoutRAA = 1700\n",
      "[Automatic] N_MCP of RtSwithoutRAA = 1930\n",
      "[Manual] N_MCP of RtSwithRAA = 914\n",
      "[Automatic] N_MCP of RtSwithRAA = 1009\n",
      "[Manual] N_MCP of monologue task = 4682\n",
      "[Automatic] N_MCP of monologue task = 5058\n",
      "[Manual] N_MCP of WoZ_Interview = 1913\n",
      "[Automatic] N_MCP of WoZ_Interview = 2069\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CI>\")\n",
    "\n",
    "    print(f\"[Manual] N_MCP of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_MCP of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<CI>\")\n",
    "print(f\"[Manual] N_MCP of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_MCP of monologue task = {sum(n_tags_auto)}\")\n",
    "\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CI>\")\n",
    "\n",
    "    print(f\"[Manual] N_MCP of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_MCP of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts ECP tags in beginners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_ECP of Arg_Oly = 268\n",
      "[Automatic] N_ECP of Arg_Oly = 217\n",
      "[Manual] N_ECP of Cartoon = 232\n",
      "[Automatic] N_ECP of Cartoon = 193\n",
      "[Manual] N_ECP of RtSwithoutRAA = 402\n",
      "[Automatic] N_ECP of RtSwithoutRAA = 378\n",
      "[Manual] N_ECP of RtSwithRAA = 211\n",
      "[Automatic] N_ECP of RtSwithRAA = 168\n",
      "[Manual] N_ECP of monologue task = 1113\n",
      "[Automatic] N_ECP of monologue task = 956\n",
      "[Manual] N_ECP of WoZ_Interview = 472\n",
      "[Automatic] N_ECP of WoZ_Interview = 443\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CE>\")\n",
    "\n",
    "    print(f\"[Manual] N_ECP of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_ECP of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<CE>\")\n",
    "print(f\"[Manual] N_ECP of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_ECP of monologue task = {sum(n_tags_auto)}\")\n",
    "\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CE>\")\n",
    "\n",
    "    print(f\"[Manual] N_ECP of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_ECP of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.2. Intemediate\n",
    "\n",
    "The following code block loads intermediate group's speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "intemediate_dataset = load_dataset(rating_filter_monologue=[3, 4, 5], rating_filter_dialogue=[2, 3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculates WER of intemediate learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of Arg_Oly = 0.15636908002177463\n",
      "WER of Cartoon = 0.13635947652603642\n",
      "WER of RtSwithoutRAA = 0.20490001047010784\n",
      "WER of RtSwithRAA = 0.20781302733715828\n",
      "WER of monologue task = 0.17790715257825496\n",
      "WER of WoZ_Interview = 0.14034321645342998\n",
      "WER of all tasks = 0.16416050112657601\n"
     ]
    }
   ],
   "source": [
    "monologue_data = []\n",
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {monologue_task} = {res}\")\n",
    "\n",
    "    monologue_data += annotation_results\n",
    "\n",
    "res = calculate_wer(monologue_data, remove_filer=True)\n",
    "print(f\"WER of monologue task = {res}\")\n",
    "\n",
    "dialogue_data = []\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {dialogue_task} = {res}\")\n",
    "\n",
    "    dialogue_data += annotation_results\n",
    "\n",
    "all_task_data = monologue_data + dialogue_data\n",
    "\n",
    "res = calculate_wer(all_task_data, remove_filer=True)\n",
    "print(f\"WER of all tasks = {res}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts disfluency tags in intemediate learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_disfluency of Arg_Oly = 958\n",
      "[Automatic] N_disfluency of Arg_Oly = 938\n",
      "[Manual] N_disfluency of Cartoon = 1729\n",
      "[Automatic] N_disfluency of Cartoon = 1595\n",
      "[Manual] N_disfluency of RtSwithoutRAA = 1446\n",
      "[Automatic] N_disfluency of RtSwithoutRAA = 1222\n",
      "[Manual] N_disfluency of RtSwithRAA = 1801\n",
      "[Automatic] N_disfluency of RtSwithRAA = 1611\n",
      "[Manual] N_disfluency of monologue task = 5934\n",
      "[Automatic] N_disfluency of monologue task = 5366\n",
      "[Manual] N_disfluency of WoZ_Interview = 2662\n",
      "[Automatic] N_disfluency of WoZ_Interview = 2439\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<DISFLUENCY>\")\n",
    "\n",
    "    print(f\"[Manual] N_disfluency of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_disfluency of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<DISFLUENCY>\")\n",
    "print(f\"[Manual] N_disfluency of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_disfluency of monologue task = {sum(n_tags_auto)}\")\n",
    "\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<DISFLUENCY>\")\n",
    "\n",
    "    print(f\"[Manual] N_disfluency of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_disfluency of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts MCP tags in intemediate learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_MCP of Arg_Oly = 2268\n",
      "[Automatic] N_MCP of Arg_Oly = 2644\n",
      "[Manual] N_MCP of Cartoon = 3622\n",
      "[Automatic] N_MCP of Cartoon = 4432\n",
      "[Manual] N_MCP of RtSwithoutRAA = 3061\n",
      "[Automatic] N_MCP of RtSwithoutRAA = 3720\n",
      "[Manual] N_MCP of RtSwithRAA = 4001\n",
      "[Automatic] N_MCP of RtSwithRAA = 4746\n",
      "[Manual] N_MCP of monologue task = 12952\n",
      "[Automatic] N_MCP of monologue task = 15542\n",
      "[Manual] N_MCP of WoZ_Interview = 4911\n",
      "[Automatic] N_MCP of WoZ_Interview = 7022\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CI>\")\n",
    "\n",
    "    print(f\"[Manual] N_MCP of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_MCP of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<CI>\")\n",
    "print(f\"[Manual] N_MCP of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_MCP of monologue task = {sum(n_tags_auto)}\")\n",
    "\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CI>\")\n",
    "\n",
    "    print(f\"[Manual] N_MCP of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_MCP of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts ECP tags in intermediate learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_ECP of Arg_Oly = 608\n",
      "[Automatic] N_ECP of Arg_Oly = 534\n",
      "[Manual] N_ECP of Cartoon = 1186\n",
      "[Automatic] N_ECP of Cartoon = 1065\n",
      "[Manual] N_ECP of RtSwithoutRAA = 955\n",
      "[Automatic] N_ECP of RtSwithoutRAA = 809\n",
      "[Manual] N_ECP of RtSwithRAA = 1154\n",
      "[Automatic] N_ECP of RtSwithRAA = 1005\n",
      "[Manual] N_ECP of monologue task = 3903\n",
      "[Automatic] N_ECP of monologue task = 3413\n",
      "[Manual] N_ECP of WoZ_Interview = 1682\n",
      "[Automatic] N_ECP of WoZ_Interview = 1813\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CE>\")\n",
    "\n",
    "    print(f\"[Manual] N_ECP of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_ECP of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<CE>\")\n",
    "print(f\"[Manual] N_ECP of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_ECP of monologue task = {sum(n_tags_auto)}\")\n",
    "\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CE>\")\n",
    "\n",
    "    print(f\"[Manual] N_ECP of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_ECP of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.3. Advanced \n",
    "\n",
    "The following code block loads advanced learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "advanced_dataset = load_dataset(rating_filter_monologue=[6, 7, 8], rating_filter_dialogue=[4, 5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculates WER of advanced learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of Arg_Oly = 0.13485070974057758\n",
      "WER of Cartoon = 0.11321073055508689\n",
      "WER of RtSwithoutRAA = 0.15263628239499552\n",
      "WER of RtSwithRAA = 0.16176742466259939\n",
      "WER of monologue task = 0.14041014416900605\n",
      "WER of WoZ_Interview = 0.081675562024907\n",
      "WER of all tasks = 0.12705882352941175\n"
     ]
    }
   ],
   "source": [
    "monologue_data = []\n",
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {monologue_task} = {res}\")\n",
    "\n",
    "    monologue_data += annotation_results\n",
    "\n",
    "res = calculate_wer(monologue_data, remove_filer=True)\n",
    "print(f\"WER of monologue task = {res}\")\n",
    "\n",
    "dialogue_data = []\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {dialogue_task} = {res}\")\n",
    "\n",
    "    dialogue_data += annotation_results\n",
    "\n",
    "all_task_data = monologue_data + dialogue_data\n",
    "\n",
    "res = calculate_wer(all_task_data, remove_filer=True)\n",
    "print(f\"WER of all tasks = {res}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts disfluency tags in advanced learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_disfluency of Arg_Oly = 309\n",
      "[Automatic] N_disfluency of Arg_Oly = 303\n",
      "[Manual] N_disfluency of Cartoon = 698\n",
      "[Automatic] N_disfluency of Cartoon = 571\n",
      "[Manual] N_disfluency of RtSwithoutRAA = 686\n",
      "[Automatic] N_disfluency of RtSwithoutRAA = 612\n",
      "[Manual] N_disfluency of RtSwithRAA = 712\n",
      "[Automatic] N_disfluency of RtSwithRAA = 576\n",
      "[Manual] N_disfluency of monologue task = 2405\n",
      "[Automatic] N_disfluency of monologue task = 2062\n",
      "[Manual] N_disfluency of WoZ_Interview = 318\n",
      "[Automatic] N_disfluency of WoZ_Interview = 345\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<DISFLUENCY>\")\n",
    "\n",
    "    print(f\"[Manual] N_disfluency of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_disfluency of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<DISFLUENCY>\")\n",
    "print(f\"[Manual] N_disfluency of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_disfluency of monologue task = {sum(n_tags_auto)}\")\n",
    "\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<DISFLUENCY>\")\n",
    "\n",
    "    print(f\"[Manual] N_disfluency of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_disfluency of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts MCP tags in advanced learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_MCP of Arg_Oly = 778\n",
      "[Automatic] N_MCP of Arg_Oly = 996\n",
      "[Manual] N_MCP of Cartoon = 1125\n",
      "[Automatic] N_MCP of Cartoon = 1471\n",
      "[Manual] N_MCP of RtSwithoutRAA = 1408\n",
      "[Automatic] N_MCP of RtSwithoutRAA = 1717\n",
      "[Manual] N_MCP of RtSwithRAA = 1417\n",
      "[Automatic] N_MCP of RtSwithRAA = 1747\n",
      "[Manual] N_MCP of monologue task = 4728\n",
      "[Automatic] N_MCP of monologue task = 5931\n",
      "[Manual] N_MCP of WoZ_Interview = 750\n",
      "[Automatic] N_MCP of WoZ_Interview = 1231\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CI>\")\n",
    "\n",
    "    print(f\"[Manual] N_MCP of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_MCP of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<CI>\")\n",
    "print(f\"[Manual] N_MCP of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_MCP of monologue task = {sum(n_tags_auto)}\")\n",
    "\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CI>\")\n",
    "\n",
    "    print(f\"[Manual] N_MCP of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_MCP of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block counts ECP tags in advanced learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Manual] N_ECP of Arg_Oly = 266\n",
      "[Automatic] N_ECP of Arg_Oly = 250\n",
      "[Manual] N_ECP of Cartoon = 567\n",
      "[Automatic] N_ECP of Cartoon = 552\n",
      "[Manual] N_ECP of RtSwithoutRAA = 478\n",
      "[Automatic] N_ECP of RtSwithoutRAA = 440\n",
      "[Manual] N_ECP of RtSwithRAA = 477\n",
      "[Automatic] N_ECP of RtSwithRAA = 400\n",
      "[Manual] N_ECP of monologue task = 1788\n",
      "[Automatic] N_ECP of monologue task = 1642\n",
      "[Manual] N_ECP of WoZ_Interview = 260\n",
      "[Automatic] N_ECP of WoZ_Interview = 390\n"
     ]
    }
   ],
   "source": [
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CE>\")\n",
    "\n",
    "    print(f\"[Manual] N_ECP of {monologue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_ECP of {monologue_task} = {sum(n_tags_auto)}\")\n",
    "\n",
    "n_tags_manu, n_tags_auto = count_tags(monologue_data, \"<CE>\")\n",
    "print(f\"[Manual] N_ECP of monologue task = {sum(n_tags_manu)}\")\n",
    "print(f\"[Automatic] N_ECP of monologue task = {sum(n_tags_auto)}\")\n",
    "\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    n_tags_manu, n_tags_auto = count_tags(annotation_results, \"<CE>\")\n",
    "\n",
    "    print(f\"[Manual] N_ECP of {dialogue_task} = {sum(n_tags_manu)}\")\n",
    "    print(f\"[Automatic] N_ECP of {dialogue_task} = {sum(n_tags_auto)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "teai-incremental-classifier",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}