merge_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0ddc505-bc30-4cda-97fc-34915ddc5f96",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "91d0ce10-2052-45f2-9de2-3ea4d247b801",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_merge_df(cv_path: str, gowajee_path: str, smart_home_path: str, health_path: str) -> pd.DataFrame:\n",
    "    cv_df = pd.read_csv(cv_path)\n",
    "    cv_df[\"path\"] = \"/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips/\" + cv_df[\"path\"]\n",
    "\n",
    "    gowajee_df = pd.read_csv(gowajee_path)\n",
    "    gowajee_df[\"path\"] = \"/mnt/d/data/gowajee_v0.9.2/v0.9.2/\" + gowajee_df[\"path\"]\n",
    "    \n",
    "    smart_home_df = pd.read_csv(smart_home_path)\n",
    "    smart_home_df[\"path\"] = \"/mnt/d/data/Dataset/Smarthome/Record/\" + smart_home_df[\"path\"]\n",
    "    \n",
    "    health_df = pd.read_csv(health_path)\n",
    "    health_df[\"path\"] = \"/mnt/d/data/Dataset/Healthcare/Record/\" + health_df[\"path\"]\n",
    "\n",
    "    df = pd.concat([cv_df, gowajee_df, smart_home_df, health_df])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "544aa3f4-1ffc-403d-a5c0-19e5288024f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = get_merge_df(cv_path=\"train.csv\", gowajee_path=\"gowajee_train.csv\", smart_home_path=\"smart_home_train.csv\", health_path=\"health_train.csv\")\n",
    "dev = get_merge_df(cv_path=\"dev.csv\", gowajee_path=\"gowajee_dev.csv\", smart_home_path=\"smart_home_dev.csv\", health_path=\"health_dev.csv\")\n",
    "test = get_merge_df(cv_path=\"test.csv\", gowajee_path=\"gowajee_test.csv\", smart_home_path=\"smart_home_test.csv\", health_path=\"health_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7435a347-c53c-4443-bb52-5fd24fa43619",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv(\"train_merge.csv\", index=False)\n",
    "dev.to_csv(\"dev_merge.csv\", index=False)\n",
    "test.to_csv(\"test_merge.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "064e1b5a-45eb-410d-af49-f403b13a8629",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>ใครเป็นผู้รับ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>ใช่ในห้องอาหาร</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>การที่จะทำเค้กแต่งงานชั้นเลิศคุณจะต้องใช้น้ำตา...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>เขาหยุดแล้วคว้ามือออกมาเฉียดตัวฉันไปเส้นยาแดงผ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>ผู้ชายคือช้างเท้าหน้าแต่ผู้หญิงคือควาญช้าง</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7675</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>ปวดท้องเหมือนลำไส้อักเสบ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7676</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>รู้สึกอักเสบบริเวณช่องปาก</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7677</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>เธอเจ็บเข่าจนเดินไม่ไหว</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7678</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>ฉันรู้สึกคอแห้งเพราะฉันไอทั้งวัน</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7679</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>ที่มือขึ้นตุ่มเพราะถือของหนักเกินไป</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>166250 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   path  \\\n",
       "0     /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "1     /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "2     /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "3     /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "4     /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "...                                                 ...   \n",
       "7675  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "7676  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "7677  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "7678  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "7679  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "\n",
       "                                               sentence  \n",
       "0                                         ใครเป็นผู้รับ  \n",
       "1                                        ใช่ในห้องอาหาร  \n",
       "2     การที่จะทำเค้กแต่งงานชั้นเลิศคุณจะต้องใช้น้ำตา...  \n",
       "3     เขาหยุดแล้วคว้ามือออกมาเฉียดตัวฉันไปเส้นยาแดงผ...  \n",
       "4            ผู้ชายคือช้างเท้าหน้าแต่ผู้หญิงคือควาญช้าง  \n",
       "...                                                 ...  \n",
       "7675                           ปวดท้องเหมือนลำไส้อักเสบ  \n",
       "7676                          รู้สึกอักเสบบริเวณช่องปาก  \n",
       "7677                            เธอเจ็บเข่าจนเดินไม่ไหว  \n",
       "7678                   ฉันรู้สึกคอแห้งเพราะฉันไอทั้งวัน  \n",
       "7679                ที่มือขึ้นตุ่มเพราะถือของหนักเกินไป  \n",
       "\n",
       "[166250 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1fa1bb6d-4080-4313-880e-d60d019b648e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>กระเป๋าหนังสีน้ำตาลถูกแขวนด้วยสายของมัน</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>ซึ่งตรงนี้มีการถามถึงสองสามจุด</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>แม่หนูเกลียดและรังเกียจเขามาโดยตลอด</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>มูลนิธิศูนย์พิทักษ์สิทธิเด็ก</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>ในเมื่อการมีนายนั้นก็ดีเขาให้ทำอะไรก็ทำไปชีวิต...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>955</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>หลังจากหายจากโควิดสิบเก้ารู้สึกหายใจลำบาก</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>956</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>ฉันเจ็บตาเมื่อมีฝุ่นเข้าตา</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>957</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>เจ็บตาทั้งสองข้างเลยระบมเหมือนถูกกระแทกค่ะ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>958</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>กระจกร่วงลงมาแตกเศษกระจกเข้าตารู้สึกมองเห็นไม่...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>เมื่อไรลูกลูกจะโทรมาหาฉันซักที</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7961 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  path  \\\n",
       "0    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "1    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "2    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "3    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "4    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "..                                                 ...   \n",
       "955  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "956  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "957  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "958  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "959  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "\n",
       "                                              sentence  \n",
       "0              กระเป๋าหนังสีน้ำตาลถูกแขวนด้วยสายของมัน  \n",
       "1                       ซึ่งตรงนี้มีการถามถึงสองสามจุด  \n",
       "2                  แม่หนูเกลียดและรังเกียจเขามาโดยตลอด  \n",
       "3                         มูลนิธิศูนย์พิทักษ์สิทธิเด็ก  \n",
       "4    ในเมื่อการมีนายนั้นก็ดีเขาให้ทำอะไรก็ทำไปชีวิต...  \n",
       "..                                                 ...  \n",
       "955          หลังจากหายจากโควิดสิบเก้ารู้สึกหายใจลำบาก  \n",
       "956                         ฉันเจ็บตาเมื่อมีฝุ่นเข้าตา  \n",
       "957         เจ็บตาทั้งสองข้างเลยระบมเหมือนถูกกระแทกค่ะ  \n",
       "958  กระจกร่วงลงมาแตกเศษกระจกเข้าตารู้สึกมองเห็นไม่...  \n",
       "959                     เมื่อไรลูกลูกจะโทรมาหาฉันซักที  \n",
       "\n",
       "[7961 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8600b520-d5f5-4759-a7ff-8c0e173aac46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>ข้อสังเกตเพิ่มเติมเรื่องการระงับการเข้าถึงเว็บ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>เอาออกหายทันที</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>โรงพยาบาลมหาสารคาม</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>จะประสบความสำเร็จ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...</td>\n",
       "      <td>โอวชอบเหมือนเราเลย</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>955</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>เมื่อเรารู้สึกไม่สบายเกิดจากร่างกายขาดสารอาหาร</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>956</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>ฉันปวดท้องจุกเสียดแน่น</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>957</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>ปวดหัวเหมือนมีก้อนหินในหัว</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>958</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>เครียดจากการทำงาน</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959</th>\n",
       "      <td>/mnt/d/data/Dataset/Healthcare/Record/record_h...</td>\n",
       "      <td>เขารู้สึกเจ็บตาเหมือนเป็นตากุ้งยิง</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5797 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  path  \\\n",
       "0    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "1    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "2    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "3    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "4    /mnt/d/data/cv-corpus-13.0-2023-03-09/th/clips...   \n",
       "..                                                 ...   \n",
       "955  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "956  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "957  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "958  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "959  /mnt/d/data/Dataset/Healthcare/Record/record_h...   \n",
       "\n",
       "                                              sentence  \n",
       "0    ข้อสังเกตเพิ่มเติมเรื่องการระงับการเข้าถึงเว็บ...  \n",
       "1                                       เอาออกหายทันที  \n",
       "2                                   โรงพยาบาลมหาสารคาม  \n",
       "3                                    จะประสบความสำเร็จ  \n",
       "4                                   โอวชอบเหมือนเราเลย  \n",
       "..                                                 ...  \n",
       "955     เมื่อเรารู้สึกไม่สบายเกิดจากร่างกายขาดสารอาหาร  \n",
       "956                             ฉันปวดท้องจุกเสียดแน่น  \n",
       "957                         ปวดหัวเหมือนมีก้อนหินในหัว  \n",
       "958                                  เครียดจากการทำงาน  \n",
       "959                 เขารู้สึกเจ็บตาเหมือนเป็นตากุ้งยิง  \n",
       "\n",
       "[5797 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}