05_Experiment_11_Preliminary_Analysis_ASR_Whisper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5/14 (Tue) | Experiment\n",
    "\n",
    "# Preliminary Analyses of Annoation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Introduction\n",
    "\n",
    "This notebook conducts preliminary analyses.\n",
    "The goal of current analyses is to fill the following table.\n",
    "\n",
    "| Task | WER | N disfluency (Manual / Automatic) | N MCP (Manual / Automatic) | N ECP (Manual / Automatic) |\n",
    "| - | - | - | - | - |\n",
    "| Arg_Oly |  |  |  |  |\n",
    "| Cartoon |  |  |  |  |\n",
    "| RtSwithoutRAA |  |  |  |  |\n",
    "| RtSwithRAA |  |  |  |  |\n",
    "| Monologue |  |  |  |  |\n",
    "| WoZ_Interview |  |  |  |  |\n",
    "| ALL |  |  |  |  |\n",
    "\n",
    "Before starting the analyses, the following code block loads required packages and define global variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List, Tuple, Dict, Generator, Optional\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from jiwer import wer\n",
    "\n",
    "from utils.mfr import logit_2_rating\n",
    "\n",
    "DATA_DIR = Path(\"/home/matsuura/Development/app/feature_extraction_api/experiment/data\")\n",
    "\n",
    "MONOLOGUE_TASK = [\"Arg_Oly\", \"Cartoon\", \"RtSwithoutRAA\", \"RtSwithRAA\"]\n",
    "DIALOGUE_TASK = [\"WoZ_Interview\"]\n",
    "\n",
    "FILLER = {\"uh\", \"ah\", \"um\", \"mm\", \"hmm\", \"oh\", \"mm-hmm\", \"er\", \"mhm\", \"uh-huh\", \"er\", \"erm\", \"huh\", \"uhu\", \"mmhmm\", \"uhhuh\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## 2. Define Functions\n",
    "\n",
    "This section defines functions for the preliminary analyses.\n",
    "The following code block defines two functions; one generates csv file paths of manual and automatic annotation results; and another one loads them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def annotation_result_csv_path_generator(\n",
    "        task: str, \n",
    "        asr_service: str =\"rev\",\n",
    "        rating_filter: Optional[List[int]] =None\n",
    ") -> Generator[Tuple[Path, Path], None, None]:\n",
    "    load_dir = DATA_DIR / f\"{task}/10_SCTK_Inputs\"\n",
    "\n",
    "    if rating_filter is None:\n",
    "        for manu_csv_path in load_dir.glob(\"*_manu.csv\"):\n",
    "            filename = manu_csv_path.stem.removesuffix(\"_manu\")\n",
    "\n",
    "            if asr_service == \"rev\":\n",
    "                auto_csv_path = load_dir / f\"{filename}_auto.csv\"\n",
    "            elif asr_service == \"whisper\":\n",
    "                auto_csv_path = DATA_DIR / f\"{task}/14_ASR_Whisper/{filename}_auto.csv\"\n",
    "            elif asr_service == \"google\":\n",
    "                if task in MONOLOGUE_TASK:\n",
    "                    auto_csv_path = DATA_DIR / f\"{task}/13_ASR_Google/{filename}_auto.csv\"\n",
    "                else:\n",
    "                    manu_csv_path = DATA_DIR / f\"{task}/01_Manual_TextGrid/{filename[:3]}.csv\"\n",
    "                    auto_csv_path = DATA_DIR / f\"{task}/13_ASR_Google/{filename[:3]}.csv\"\n",
    "\n",
    "            yield manu_csv_path, auto_csv_path\n",
    "    else:\n",
    "        pf_path = DATA_DIR / f\"{task}/12_PF_Rating/pf_rating.csv\"\n",
    "        df_pf = pd.read_csv(pf_path)\n",
    "        uid_list = df_pf[\"uid\"].to_numpy()\n",
    "\n",
    "        logit_path = pf_path.parent / \"logit.csv\"\n",
    "        threshold_path = logit_path.parent / \"threshold.csv\"\n",
    "        \n",
    "        df_logit = pd.read_csv(logit_path, index_col=0)\n",
    "        rating_list = logit_2_rating(df_logit[\"theta\"], threshold_path)\n",
    "\n",
    "        mask = np.full(rating_list.shape, False, dtype=bool)\n",
    "        for rating in rating_filter:\n",
    "            mask = mask | (rating_list == rating)\n",
    "        \n",
    "        uid_list = uid_list[mask]\n",
    "\n",
    "        for uid in uid_list:\n",
    "            if task == \"WoZ_Interview\":\n",
    "                uid = str(int(uid)).zfill(3)\n",
    "\n",
    "            filename_pattern = f\"{uid}*_manu.csv\"\n",
    "            for manu_csv_path in load_dir.glob(filename_pattern):\n",
    "                filename = manu_csv_path.stem.removesuffix(\"_manu\")\n",
    "                \n",
    "                if asr_service == \"rev\":\n",
    "                    auto_csv_path = load_dir / f\"{filename}_auto.csv\"\n",
    "                elif asr_service == \"whisper\":\n",
    "                    auto_csv_path = DATA_DIR / f\"{task}/14_ASR_Whisper/{filename}_auto.csv\"\n",
    "                elif asr_service == \"google\":\n",
    "                    auto_csv_path = DATA_DIR / f\"{task}/13_ASR_Google/{filename}_auto.csv\"\n",
    "\n",
    "                yield manu_csv_path, auto_csv_path\n",
    "\n",
    "def load_dataset(\n",
    "        asr_service: str =\"rev\",\n",
    "        rating_filter_monologue: Optional[List[int]] =None,\n",
    "        rating_filter_dialogue: Optional[List[int]] =None,\n",
    ") -> Dict[str, Dict[str, List[Dict[str, pd.DataFrame]]]]:\n",
    "    dataset = {\n",
    "        \"monologue\": {},\n",
    "        \"dialogue\": {}\n",
    "    }\n",
    "    \n",
    "    for monologue_task in MONOLOGUE_TASK:\n",
    "        dataset[\"monologue\"][monologue_task] = []\n",
    "        \n",
    "        for manu_csv_path, auto_csv_path in annotation_result_csv_path_generator(monologue_task, asr_service=asr_service, rating_filter=rating_filter_monologue):\n",
    "            df_manu = pd.read_csv(manu_csv_path)\n",
    "            df_auto = pd.DataFrame([], columns=[\"text\"])\n",
    "            if auto_csv_path.exists():\n",
    "                df_auto = pd.read_csv(auto_csv_path)\n",
    "\n",
    "            dataset[\"monologue\"][monologue_task].append({\n",
    "                \"manual\": df_manu,\n",
    "                \"automatic\": df_auto\n",
    "            })\n",
    "\n",
    "    for dialogue_task in DIALOGUE_TASK:\n",
    "        dataset[\"dialogue\"][dialogue_task] = []\n",
    "\n",
    "        for manu_csv_path, auto_csv_path in annotation_result_csv_path_generator(dialogue_task, asr_service=asr_service, rating_filter=rating_filter_dialogue):\n",
    "            df_manu = pd.read_csv(manu_csv_path, na_values=[\"\", \" \"], keep_default_na=False)\n",
    "            df_auto = pd.DataFrame([], columns=[\"text\"])\n",
    "            if auto_csv_path.exists():\n",
    "                df_auto = pd.read_csv(auto_csv_path, na_values=[\"\", \" \"], keep_default_na=False)\n",
    "\n",
    "            dataset[\"dialogue\"][dialogue_task].append({\n",
    "                \"manual\": df_manu,\n",
    "                \"automatic\": df_auto\n",
    "            })\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block defines a function to calculate WER."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_google_dialog(df_manu: pd.DataFrame, df_auto: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:\n",
    "    mask_manu = (df_manu[\"speaker\"] == \"user\") & (df_manu[\"topic\"] != \"intro\") & (df_manu[\"topic\"] != \"closing\")\n",
    "    mask_auto = (df_auto[\"speaker\"] == \"user\") & (df_auto[\"phase\"] != \"intro\") & (df_auto[\"phase\"] != \"closing\")\n",
    "\n",
    "    df_manu_masked = df_manu[mask_manu]\n",
    "    df_auto_masked = df_auto[mask_auto]\n",
    "\n",
    "    text_manu = \" \".join(df_manu_masked[\"transcript\"])\n",
    "    text_auto = \" \".join(df_auto_masked[\"text\"])\n",
    "\n",
    "    while \"  \" in text_manu:\n",
    "        text_manu = text_manu.replace(\"  \", \" \")\n",
    "    while \"  \" in text_auto:\n",
    "        text_auto = text_auto.replace(\"  \", \" \")\n",
    "    \n",
    "    if text_manu[0] == \" \":\n",
    "        text_manu = text_manu[1:]\n",
    "    if text_auto[0] == \" \":\n",
    "        text_auto = text_auto[1:]\n",
    "\n",
    "    if text_manu[-1] == \" \":\n",
    "        text_manu = text_manu[:-1]\n",
    "    if text_manu[-1] == \" \":\n",
    "        text_manu = text_manu[:-1]\n",
    "\n",
    "    token_manu = np.array([text_manu.split(\" \"), text_manu.split(\" \")]).T\n",
    "    token_auto = np.array([text_auto.split(\" \"), text_auto.split(\" \")]).T\n",
    "\n",
    "    df_manu = pd.DataFrame(token_manu, columns=[\"text\", \"hoge\"])\n",
    "    df_auto = pd.DataFrame(token_auto, columns=[\"text\", \"hoge\"])\n",
    "\n",
    "    return df_manu, df_auto\n",
    "\n",
    "def calculate_wer(annotation_results: List[Dict[str, pd.DataFrame]], remove_filer: bool =False, google_dialog: bool =False) -> float:\n",
    "    ref = []\n",
    "    hyp = []\n",
    "\n",
    "    for annotation_result in annotation_results:\n",
    "        df_manu = annotation_result[\"manual\"]\n",
    "        df_auto = annotation_result[\"automatic\"]\n",
    "\n",
    "        if google_dialog:\n",
    "            df_manu, df_auto = convert_google_dialog(df_manu, df_auto)\n",
    "\n",
    "        mask_tag_manu = df_manu[\"text\"].astype(str).str.endswith(\">\")\n",
    "        mask_tag_auto = df_auto[\"text\"].astype(str).str.endswith(\">\")\n",
    "\n",
    "        df_manu = df_manu[~mask_tag_manu]\n",
    "        df_auto = df_auto[~mask_tag_auto]\n",
    "\n",
    "        if remove_filer:\n",
    "            for filler in FILLER:\n",
    "                mask_filler_manu = (df_manu[\"text\"] == filler)\n",
    "                df_manu = df_manu[~mask_filler_manu]\n",
    "\n",
    "                mask_filler_auto = (df_auto[\"text\"] == filler)\n",
    "                df_auto = df_auto[~mask_filler_auto]\n",
    "\n",
    "        text_manu = \" \".join(df_manu[\"text\"].astype(str))\n",
    "        text_auto = \" \".join(df_auto[\"text\"].astype(str))\n",
    "\n",
    "        if len(text_manu) == 0 or len(text_auto) == 0:\n",
    "            continue\n",
    "\n",
    "        ref.append(text_manu)\n",
    "        hyp.append(text_auto)\n",
    "\n",
    "    return wer(ref, hyp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## 3. Preliminary Analyses (Whisper)\n",
    "\n",
    "This section conducts the preliminary analyses.\n",
    "The following code block loads entire dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_dataset(asr_service=\"whisper\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1. WER\n",
    "\n",
    "The following code block calculate WER of monologue tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of Arg_Oly = 0.19619777063530056\n",
      "WER of Cartoon = 0.19581162963750398\n",
      "WER of RtSwithoutRAA = 0.23387767838568754\n",
      "WER of RtSwithRAA = 0.2385330632174952\n",
      "WER of monologue task = 0.21750375438011013\n"
     ]
    }
   ],
   "source": [
    "monologue_data = []\n",
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {monologue_task} = {res}\")\n",
    "\n",
    "    monologue_data += annotation_results\n",
    "\n",
    "res = calculate_wer(monologue_data, remove_filer=True)\n",
    "print(f\"WER of monologue task = {res}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculate WER of a dialogue task."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of WoZ_Interview = 0.17853543836076588\n"
     ]
    }
   ],
   "source": [
    "dialogue_data = []\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {dialogue_task} = {res}\")\n",
    "\n",
    "    dialogue_data += annotation_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calcualte WER of the entire tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of all tasks = 0.20457079152731328\n"
     ]
    }
   ],
   "source": [
    "all_task_data = monologue_data + dialogue_data\n",
    "\n",
    "res = calculate_wer(all_task_data, remove_filer=True)\n",
    "print(f\"WER of all tasks = {res}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## 5. Additional Analyses\n",
    "\n",
    "This section conducts the same analyses for each PF groups.\n",
    "\n",
    "### 5.1. Beginners\n",
    "\n",
    "The following code block loads beginners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "beginner_dataset = load_dataset(asr_service=\"whisper\", rating_filter_monologue=[0, 1, 2], rating_filter_dialogue=[0, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculates WER of beginners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of Arg_Oly = 0.279711508309815\n",
      "WER of Cartoon = 0.28061224489795916\n",
      "WER of RtSwithoutRAA = 0.30720235178833905\n",
      "WER of RtSwithRAA = 0.27638190954773867\n",
      "WER of monologue task = 0.28927903038944836\n",
      "WER of WoZ_Interview = 0.2614270629054156\n",
      "WER of all tasks = 0.2789355742296919\n"
     ]
    }
   ],
   "source": [
    "monologue_data = []\n",
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {monologue_task} = {res}\")\n",
    "\n",
    "    monologue_data += annotation_results\n",
    "\n",
    "res = calculate_wer(monologue_data, remove_filer=True)\n",
    "print(f\"WER of monologue task = {res}\")\n",
    "\n",
    "dialogue_data = []\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = beginner_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {dialogue_task} = {res}\")\n",
    "\n",
    "    dialogue_data += annotation_results\n",
    "\n",
    "all_task_data = monologue_data + dialogue_data\n",
    "\n",
    "res = calculate_wer(all_task_data, remove_filer=True)\n",
    "print(f\"WER of all tasks = {res}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.2. Intemediate\n",
    "\n",
    "The following code block loads intermediate group's speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "intemediate_dataset = load_dataset(asr_service=\"whisper\", rating_filter_monologue=[3, 4, 5], rating_filter_dialogue=[2, 3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculates WER of intemediate learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of Arg_Oly = 0.18277082199237887\n",
      "WER of Cartoon = 0.20087855770110735\n",
      "WER of RtSwithoutRAA = 0.23568212752591353\n",
      "WER of RtSwithRAA = 0.2473844076949038\n",
      "WER of monologue task = 0.21979434447300772\n",
      "WER of WoZ_Interview = 0.17749749050757213\n",
      "WER of all tasks = 0.20431052387723475\n"
     ]
    }
   ],
   "source": [
    "monologue_data = []\n",
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {monologue_task} = {res}\")\n",
    "\n",
    "    monologue_data += annotation_results\n",
    "\n",
    "res = calculate_wer(monologue_data, remove_filer=True)\n",
    "print(f\"WER of monologue task = {res}\")\n",
    "\n",
    "dialogue_data = []\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = intemediate_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {dialogue_task} = {res}\")\n",
    "\n",
    "    dialogue_data += annotation_results\n",
    "\n",
    "all_task_data = monologue_data + dialogue_data\n",
    "\n",
    "res = calculate_wer(all_task_data, remove_filer=True)\n",
    "print(f\"WER of all tasks = {res}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.3. Advanced \n",
    "\n",
    "The following code block loads advanced learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "advanced_dataset = load_dataset(asr_service=\"whisper\", rating_filter_monologue=[6, 7, 8], rating_filter_dialogue=[4, 5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculates WER of advanced learners' speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER of Arg_Oly = 0.15516397454723446\n",
      "WER of Cartoon = 0.15842753500927956\n",
      "WER of RtSwithoutRAA = 0.1773011617515639\n",
      "WER of RtSwithRAA = 0.20521353300055464\n",
      "WER of monologue task = 0.17485844792310987\n",
      "WER of WoZ_Interview = 0.09349725008087997\n",
      "WER of all tasks = 0.15636604286922312\n"
     ]
    }
   ],
   "source": [
    "monologue_data = []\n",
    "for monologue_task in MONOLOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"monologue\"][monologue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {monologue_task} = {res}\")\n",
    "\n",
    "    monologue_data += annotation_results\n",
    "\n",
    "res = calculate_wer(monologue_data, remove_filer=True)\n",
    "print(f\"WER of monologue task = {res}\")\n",
    "\n",
    "dialogue_data = []\n",
    "for dialogue_task in DIALOGUE_TASK:\n",
    "    annotation_results = advanced_dataset[\"dialogue\"][dialogue_task]\n",
    "\n",
    "    res = calculate_wer(annotation_results, remove_filer=True)\n",
    "\n",
    "    print(f\"WER of {dialogue_task} = {res}\")\n",
    "\n",
    "    dialogue_data += annotation_results\n",
    "\n",
    "all_task_data = monologue_data + dialogue_data\n",
    "\n",
    "res = calculate_wer(all_task_data, remove_filer=True)\n",
    "print(f\"WER of all tasks = {res}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "teai-incremental-classifier",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}