tutorials/1.speech_to_text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install jiwer -qq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install git+https://github.com/redapesolutions/suara-kami-community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sk import predict\n",
    "from pathlib import Path\n",
    "import jiwer\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using predict function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict function is a generic function that will handle all type of input. \n",
    "\n",
    "It is a high layer api for transcription, if want to do more flexible transcription can use SK class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict single or multiple file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded model: /home/ubuntu/.sk/models/conformer_small.onnx ['CPUExecutionProvider']\n",
      "Total input path: 1\n",
      "Total audio found(.wav): 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  6.60it/s]\n"
     ]
    }
   ],
   "source": [
    "fn = \"/content/test/youtube/0228444ff34081eda587c0ca53712486.wav\" # or [\"/content/audio1.wav\",\"/content/audio2.wav\"]\n",
    "out = predict(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pada peringkat ini pembukaan sekolah hanya melibatkan'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Path(fn).with_suffix(\".txt\").read_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'texts': ['pada peringkat ini pembukaan sekolah hanya melibatkan'],\n",
       " 'filenames': [PosixPath('/content/test/youtube/0228444ff34081eda587c0ca53712486.wav')],\n",
       " 'entropy': [0.081219584],\n",
       " 'timestamps': [[0]],\n",
       " 'speakers': [['not enabled']]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict from single or multiple folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded model: /home/ubuntu/.sk/models/conformer_small.onnx ['CPUExecutionProvider']\n",
      "Total input path: 1\n",
      "Total audio found(.wav): 1700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1700/1700 [04:45<00:00,  5.96it/s]\n"
     ]
    }
   ],
   "source": [
    "fn = \"/content/test/\" # or [\"/content/test\",\"/content/data\"]\n",
    "out = predict(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['texts', 'filenames', 'entropy', 'timestamps', 'speakers'])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = out[\"texts\"]\n",
    "files = out[\"filenames\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for i,j in zip(preds,files):\n",
    "    label = j.with_suffix('.txt').read_text()\n",
    "    data.append([j,label,i,jiwer.compute_measures(label,i)['wer'],jiwer.compute_measures([l for l in label],[ii for ii in i])['wer']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>label</th>\n",
       "      <th>pred</th>\n",
       "      <th>wer</th>\n",
       "      <th>cer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/content/test/youtube/acce76874b0d5edc51fdd5fa...</td>\n",
       "      <td>sebaik saja kita selesai makan malam</td>\n",
       "      <td>sebaik saja kita selesai makan malam</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/content/test/youtube/e8c3214529ef34af368c0439...</td>\n",
       "      <td>pentaksiran tingkatan tiga p t tiga bagi tahun...</td>\n",
       "      <td>pentaksiran tingkatan tiga pt tiga bagi tahun ...</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/content/test/youtube/6c4ee08c30a81d280ea66dd7...</td>\n",
       "      <td>tengok dekat mana</td>\n",
       "      <td>dekat mana tengok</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/content/test/youtube/223d189c07d45aa377436a41...</td>\n",
       "      <td>gil keras kepala macam mak engkau engkau</td>\n",
       "      <td>gerah kepala macam manggaum lenggau</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.294118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/content/test/youtube/87f7a2b2d1d88b235f43c28c...</td>\n",
       "      <td>pengusaha sekolah swasta antarabangsa dan pusa...</td>\n",
       "      <td>pengusaha sekolah swasta antarabangsa dan pusa...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                path  \\\n",
       "0  /content/test/youtube/acce76874b0d5edc51fdd5fa...   \n",
       "1  /content/test/youtube/e8c3214529ef34af368c0439...   \n",
       "2  /content/test/youtube/6c4ee08c30a81d280ea66dd7...   \n",
       "3  /content/test/youtube/223d189c07d45aa377436a41...   \n",
       "4  /content/test/youtube/87f7a2b2d1d88b235f43c28c...   \n",
       "\n",
       "                                               label  \\\n",
       "0               sebaik saja kita selesai makan malam   \n",
       "1  pentaksiran tingkatan tiga p t tiga bagi tahun...   \n",
       "2                                  tengok dekat mana   \n",
       "3           gil keras kepala macam mak engkau engkau   \n",
       "4  pengusaha sekolah swasta antarabangsa dan pusa...   \n",
       "\n",
       "                                                pred       wer       cer  \n",
       "0               sebaik saja kita selesai makan malam  0.000000  0.000000  \n",
       "1  pentaksiran tingkatan tiga pt tiga bagi tahun ...  0.200000  0.000000  \n",
       "2                                  dekat mana tengok  0.666667  0.800000  \n",
       "3                gerah kepala macam manggaum lenggau  0.714286  0.294118  \n",
       "4  pengusaha sekolah swasta antarabangsa dan pusa...  0.000000  0.000000  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data)\n",
    "df.columns = [\"path\",\"label\",\"pred\",\"wer\",\"cer\"]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.07793928162784017, 0.025547171006191426)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"wer\"].mean(),df[\"cer\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using SK class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Why use SK class? if want to transcribe multiple time predict will load the model and lm multiple time which is not efficient, with SK object you can call it multiple time without reloading anything again."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sk import SK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded model: /home/ubuntu/.sk/models/conformer_small.onnx ['CPUExecutionProvider']\n"
     ]
    }
   ],
   "source": [
    "asr = SK(model=\"conformer_small\",decoder=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input path: 1\n",
      "Total audio found(.wav): 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  5.07it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'texts': ['pada peringkat ini pembukaan sekolah hanya melibatkan'],\n",
       " 'filenames': ['/content/test/youtube/0228444ff34081eda587c0ca53712486.wav'],\n",
       " 'entropy': [0.081219584],\n",
       " 'timestamps': [[0]],\n",
       " 'speakers': [['not enabled']],\n",
       " 'all_logits': []}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asr.transcribe_file(fn=\"/content/test/youtube/0228444ff34081eda587c0ca53712486.wav\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'texts': 'pada peringkat ini pembukaan sekolah hanya melibatkan',\n",
       " 'filenames': array([0.00311279, 0.00430298, 0.00643921, 0.00476074, 0.00372314,\n",
       "        0.00167847, 0.00363159, 0.0072937 , 0.00421143, 0.00289917,\n",
       "        0.0022583 , 0.00341797, 0.00488281, 0.00982666, 0.0098877 ,\n",
       "        0.00900269], dtype=float32),\n",
       " 'entropy': 0.081219584,\n",
       " 'timestamps': [0]}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import librosa\n",
    "asr.transcribe_array(array=librosa.load(\"/content/test/youtube/0228444ff34081eda587c0ca53712486.wav\",16000)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "downloading: en_v5.onnx\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "114836it [00:06, 18753.09it/s]                                                                                                                                                                                                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved to: /home/ubuntu/.sk/models/en_v5.onnx\n",
      "loaded model: /home/ubuntu/.sk/models/en_v5.onnx ['CPUExecutionProvider']\n"
     ]
    }
   ],
   "source": [
    "asr = SK(model=\"silero_en\",decoder=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input path: 1\n",
      "Total audio found(.wav): 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  6.09it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'texts': [\"because if the next person doesn't do a good job at as you all your efforts may be wasted\"],\n",
       " 'filenames': ['/content/testset-imda/0.wav'],\n",
       " 'entropy': [-2.5951443],\n",
       " 'timestamps': [[0]],\n",
       " 'speakers': [['not enabled']],\n",
       " 'all_logits': []}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asr.transcribe_file(fn=\"/content/testset-imda/0.wav\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "downloading mixed-lower.binary.zip language model of size 600+MB, might take a while\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "778999it [00:34, 22601.06it/s]                                                                                                                                                                                                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved to: /home/ubuntu/.sk/lm/mixed-lower.binary.klm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Unigrams not provided and cannot be automatically determined from LM file (only arpa format). Decoding accuracy might be reduced.\n",
      "Found entries of length > 1 in alphabet. This is unusual unless style is BPE, but the alphabet was not recognized as BPE type. Is this correct?\n",
      "No known unigrams provided, decoding results might be a lot worse.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded model: /home/ubuntu/.sk/models/en_v5.onnx ['CPUExecutionProvider']\n",
      "loaded lm: en\n"
     ]
    }
   ],
   "source": [
    "asr = SK(model=\"silero_en\",decoder=\"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input path: 1\n",
      "Total audio found(.wav): 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00,  2.83it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'texts': [\"because if the next person doesn't do a good job as you all your efforts may be wasted\"],\n",
       " 'filenames': ['/content/testset-imda/0.wav'],\n",
       " 'entropy': [[-5.385753631591797,\n",
       "   -2.7172725200653076,\n",
       "   -2.7174205780029297,\n",
       "   -2.717975378036499,\n",
       "   -2.717954635620117,\n",
       "   -12.571464538574219,\n",
       "   -2.7051992416381836,\n",
       "   -5.339262962341309,\n",
       "   -2.7172226905822754,\n",
       "   -7.258273601531982,\n",
       "   -4.206550598144531,\n",
       "   -2.591944694519043,\n",
       "   -2.349553346633911,\n",
       "   -5.384856700897217,\n",
       "   -10.193320274353027,\n",
       "   -2.7152457237243652,\n",
       "   -2.7167961597442627,\n",
       "   -12.797026634216309]],\n",
       " 'timestamps': [[('because', 0.03, 0.04),\n",
       "   ('if', 0.05, 0.05),\n",
       "   ('the', 1.0, 1.01),\n",
       "   ('next', 1.02, 1.02),\n",
       "   ('person', 1.03, 1.04),\n",
       "   (\"doesn't\", 1.05, 2.01),\n",
       "   ('do', 2.01, 2.02),\n",
       "   ('a', 2.02, 2.03),\n",
       "   ('good', 2.04, 2.04),\n",
       "   ('job', 2.05, 3.0),\n",
       "   ('as', 3.04, 3.04),\n",
       "   ('you', 3.05, 3.06),\n",
       "   ('all', 4.04, 4.04),\n",
       "   ('your', 4.05, 5.0),\n",
       "   ('efforts', 5.01, 5.03),\n",
       "   ('may', 5.03, 5.04),\n",
       "   ('be', 5.05, 5.05),\n",
       "   ('wasted', 6.0, 6.03)]],\n",
       " 'speakers': [['not enabled']],\n",
       " 'all_logits': []}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asr.transcribe_file(fn=\"/content/testset-imda/0.wav\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "test",
   "language": "python",
   "name": "test"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}