05_Experiment_15_PowerAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "6/3 (Mon) | Experiment\n",
    "\n",
    "# Power Analyses of Cohen Kappa Calculation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Introduction\n",
    "\n",
    "This notebook conducts power analyses of Cohen Kappa to determine the minimun sample size.\n",
    "Before starting the evaluation, the following code block loads required packages and define global variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List, Tuple, Dict, Generator, Optional\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from jiwer import wer\n",
    "\n",
    "from utils.cohen_kappa import RCohenKappa\n",
    "\n",
    "DATA_DIR = Path(\"/home/matsuura/Development/app/feature_extraction_api/experiment/data\")\n",
    "\n",
    "TASK = [\"Arg_Oly\", \"Cartoon\", \"RtSwithoutRAA\", \"RtSwithRAA\", \"WoZ_Interview\"]\n",
    "\n",
    "FILLER = {\"uh\", \"ah\", \"um\", \"mm\", \"hmm\", \"oh\", \"mm-hmm\", \"er\", \"mhm\", \"uh-huh\", \"er\", \"erm\", \"huh\", \"uhu\", \"mmhmm\", \"uhhuh\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Define Functions\n",
    "\n",
    "This section defines functions for the power analyses.\n",
    "The following code block defines functions to load manual csv files in SCTK_inputs and counts required values, such as N_words, N_disfl, and N_pauses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_filler(df_sctk: pd.DataFrame) -> pd.DataFrame:\n",
    "    for filler in FILLER:\n",
    "        mask_filler = (df_sctk[\"text\"] == filler)\n",
    "        df_sctk = df_sctk[~mask_filler]\n",
    "\n",
    "    mask_filler = (df_sctk[\"type\"] == \"04_filler\")\n",
    "    df_sctk = df_sctk[~mask_filler]\n",
    "\n",
    "    return df_sctk\n",
    "\n",
    "def df_sctk_input_loader(task: str) -> Generator[pd.DataFrame, None, None]:\n",
    "    load_dir = DATA_DIR / f\"{task}/10_SCTK_Inputs\"\n",
    "    for manu_csv_path in load_dir.glob(\"*_manu.csv\"):\n",
    "        df_sctk = pd.read_csv(manu_csv_path)\n",
    "        yield df_sctk\n",
    "\n",
    "def count_values(task: str) -> Tuple[int, int, int, int, int]:\n",
    "    n_files = 0\n",
    "    n_word = 0\n",
    "    n_disfl = 0\n",
    "    n_mcp = 0\n",
    "    n_ecp = 0\n",
    "\n",
    "    for df_sctk in df_sctk_input_loader(task):\n",
    "        mask_disfl = df_sctk[\"text\"] == \"<DISFLUENCY>\"\n",
    "        mask_mcp = df_sctk[\"text\"] == \"<CI>\"\n",
    "        mask_ecp = df_sctk[\"text\"] == \"<CE>\"\n",
    "\n",
    "        mask_tag = (mask_disfl | mask_mcp | mask_ecp)\n",
    "\n",
    "        n_files += 1\n",
    "        n_word += len(df_sctk[~mask_tag]) \n",
    "        n_disfl += mask_disfl.sum()\n",
    "        n_mcp += mask_mcp.sum()\n",
    "        n_ecp += mask_ecp.sum()\n",
    "\n",
    "    return n_files, n_word, n_disfl, n_mcp, n_ecp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Calculate Proportions\n",
    "\n",
    "This section calculates the proportions of disfluency words, MCPs, and ECPs.\n",
    "The following code block counts values for the proportion calculation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Arg_Oly]\n",
      "\tN files = 128\n",
      "\tN words = 16680\n",
      "\tN disfl = 1876\n",
      "\t N MCPs = 4302\n",
      "\t N ECPs = 1142\n",
      "---\n",
      "[Cartoon]\n",
      "\tN files = 128\n",
      "\tN words = 20958\n",
      "\tN disfl = 2813\n",
      "\t N MCPs = 5559\n",
      "\t N ECPs = 1985\n",
      "---\n",
      "[RtSwithoutRAA]\n",
      "\tN files = 128\n",
      "\tN words = 21737\n",
      "\tN disfl = 2949\n",
      "\t N MCPs = 6169\n",
      "\t N ECPs = 1835\n",
      "---\n",
      "[RtSwithRAA]\n",
      "\tN files = 128\n",
      "\tN words = 21829\n",
      "\tN disfl = 2887\n",
      "\t N MCPs = 6332\n",
      "\t N ECPs = 1842\n",
      "---\n",
      "[WoZ_Interview]\n",
      "\tN files = 1743\n",
      "\tN words = 43467\n",
      "\tN disfl = 3935\n",
      "\t N MCPs = 7574\n",
      "\t N ECPs = 2414\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "n_files_all = 0\n",
    "n_word_all = 0\n",
    "n_disfl_all = 0\n",
    "n_mcp_all = 0\n",
    "n_ecp_all = 0\n",
    "\n",
    "for task in TASK:\n",
    "    n_files, n_word, n_disfl, n_mcp, n_ecp = count_values(task)\n",
    "\n",
    "    print(f\"[{task}]\")\n",
    "    print(f\"\\tN files = {n_files}\")\n",
    "    print(f\"\\tN words = {n_word}\")\n",
    "    print(f\"\\tN disfl = {n_disfl}\")\n",
    "    print(f\"\\t N MCPs = {n_mcp}\")\n",
    "    print(f\"\\t N ECPs = {n_ecp}\")\n",
    "    print(\"---\")\n",
    "\n",
    "    n_files_all += n_files\n",
    "    n_word_all += n_word\n",
    "    n_disfl_all += n_disfl\n",
    "    n_mcp_all += n_mcp\n",
    "    n_ecp_all += n_ecp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculates the mean number of words for files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean number of words = 55.286\n"
     ]
    }
   ],
   "source": [
    "print(f\"Mean number of words = {(n_word_all / n_files_all):.03f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block calculates the proportions of disfluency words, MCPs, and ECPs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prop disfl = 0.116\n",
      "Prop MCPs  = 0.240\n",
      "Prop ECPs  = 0.074\n"
     ]
    }
   ],
   "source": [
    "prop_disfl = n_disfl_all / n_word_all\n",
    "prop_mcp = n_mcp_all / n_word_all\n",
    "prop_ecp = n_ecp_all / n_word_all\n",
    "\n",
    "print(f\"Prop disfl = {prop_disfl:.03f}\")\n",
    "print(f\"Prop MCPs  = {prop_mcp:.03f}\")\n",
    "print(f\"Prop ECPs  = {prop_ecp:.03f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Power Analyses\n",
    "\n",
    "This section conducts power analyses and estimates the minimum sample size for Cohen Kappa calculation.\n",
    "The following code block constructs RCohenKappa class for the power analyses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "power_analyzer = RCohenKappa(debug=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block conducts a power analysis for Cohen Kappa of disfluency word detection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum sample size for disfl. detection = 150.226\n"
     ]
    }
   ],
   "source": [
    "n_disfl = power_analyzer.power_analysis(\n",
    "    kappa_null=0.3, kappa_alt=0.61, \n",
    "    props=prop_disfl,\n",
    "    alpha=0.05, beta=0.8\n",
    ")\n",
    "\n",
    "print(f\"Minimum sample size for disfl. detection = {n_disfl:.03f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following code block conducts power analyses for Cohen Kappa of MCP & ECP classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum sample size for pause location classification = 66.433\n"
     ]
    }
   ],
   "source": [
    "n_pause = power_analyzer.power_analysis(\n",
    "    kappa_null=0.3, kappa_alt=0.61, \n",
    "    props=[prop_mcp, prop_ecp, 1 - prop_mcp - prop_ecp],\n",
    "    alpha=0.05, beta=0.8\n",
    ")\n",
    "\n",
    "print(f\"Minimum sample size for pause location classification = {n_pause:.03f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "teai-incremental-classifier",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}