common_voice_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e86a377b",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install install sox\n",
    "!pip install wget\n",
    "!sudo apt-get update && sudo apt-get install sox libsox-fmt-al"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4afed35f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import argparse\n",
    "import csv\n",
    "import json\n",
    "import logging\n",
    "import multiprocessing\n",
    "import os\n",
    "import tarfile\n",
    "from multiprocessing.pool import ThreadPool\n",
    "from pathlib import Path\n",
    "from typing import List\n",
    "\n",
    "import sox\n",
    "from sox import Transformer\n",
    "import wget\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cdea7e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = argparse.ArgumentParser(description='Downloads and processes Mozilla Common Voice dataset.')\n",
    "parser.add_argument(\"--data_root\", default='CommonVoice_dataset/', type=str, help=\"Directory to store the dataset.\")\n",
    "parser.add_argument('--manifest_dir', default='./', type=str, help='Output directory for manifests')\n",
    "parser.add_argument(\"--num_workers\", default=multiprocessing.cpu_count(), type=int, help=\"Workers to process dataset.\")\n",
    "parser.add_argument('--sample_rate', default=16000, type=int, help='Sample rate')\n",
    "parser.add_argument('--files_to_process', nargs='+', default=['test.tsv', 'dev.tsv', 'train.tsv'],\n",
    "                    type=str, help='list of *.csv file names to process')\n",
    "parser.add_argument('--version', default='cv-corpus-8.0-2022-01-19',\n",
    "                    type=str, help='Version of the dataset (obtainable via https://commonvoice.mozilla.org/en/datasets')\n",
    "parser.add_argument('--language', default='tr',\n",
    "                    type=str, help='Which language to download.(default english,'\n",
    "                                   'check https://commonvoice.mozilla.org/en/datasets for more language codes')\n",
    "\n",
    "# args = parser.parse_args()\n",
    "args, unknown = parser.parse_known_args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57a476f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# UPDATE VERSION AND LANGUAGE\n",
    "version = 'cv-corpus-8.0-2022-01-19'\n",
    "language = 'cv-corpus-8.0-2022-01-19-en'\n",
    "\n",
    "COMMON_VOICE_URL = f\"https://voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com/\" \\\n",
    "                   \"{}/{}.tar.gz\".format(version, language)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45e990e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_manifest(\n",
    "        data: List[tuple],\n",
    "        output_name: str,\n",
    "        manifest_path: str):\n",
    "    output_file = Path(manifest_path) / output_name\n",
    "    output_file.parent.mkdir(exist_ok=True, parents=True)\n",
    "    with output_file.open(mode='w', encoding=\"utf-8\") as f:\n",
    "        for wav_path, duration, text in tqdm(data, total=len(data)):\n",
    "            f.write(\n",
    "                json.dumps({\n",
    "                    'audio_filepath': wav_path,\n",
    "                    \"duration\": duration,\n",
    "                    'text': text\n",
    "                },  ensure_ascii=False) + '\\n'\n",
    "            )\n",
    "\n",
    "def process_files(csv_file, data_root, num_workers):\n",
    "    \"\"\" Read *.csv file description, convert mp3 to wav, process text.\n",
    "        Save results to data_root.\n",
    "    Args:\n",
    "        csv_file: str, path to *.csv file with data description, usually start from 'cv-'\n",
    "        data_root: str, path to dir to save results; wav/ dir will be created\n",
    "    \"\"\"\n",
    "    wav_dir = os.path.join(data_root, 'wav/')\n",
    "    os.makedirs(wav_dir, exist_ok=True)\n",
    "    audio_clips_path = os.path.dirname(csv_file) + '/clips/'\n",
    "\n",
    "    def process(x):\n",
    "        file_path, text = x\n",
    "        file_name = os.path.splitext(os.path.basename(file_path))[0]\n",
    "        text = text.lower().strip()\n",
    "        audio_path = os.path.join(audio_clips_path, file_path)\n",
    "        output_wav_path = os.path.join(wav_dir, file_name + '.wav')\n",
    "\n",
    "        tfm = Transformer()\n",
    "        tfm.rate(samplerate=args.sample_rate)\n",
    "        tfm.build(\n",
    "            input_filepath=audio_path,\n",
    "            output_filepath=output_wav_path\n",
    "        )\n",
    "        duration = sox.file_info.duration(output_wav_path)\n",
    "        return output_wav_path, duration, text\n",
    "\n",
    "    logging.info('Converting mp3 to wav for {}.'.format(csv_file))\n",
    "    with open(csv_file, encoding=\"utf8\") as csvfile:\n",
    "        reader = csv.DictReader(csvfile, delimiter='\\t')\n",
    "        next(reader, None)  # skip the headers\n",
    "        data = [(row['path'], row['sentence']) for row in reader]\n",
    "        with ThreadPool(num_workers) as pool:\n",
    "            data = list(tqdm(pool.imap(process, data), total=len(data)))\n",
    "\n",
    "    print(data)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fc148b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    data_root = args.data_root\n",
    "    os.makedirs(data_root, exist_ok=True)\n",
    "\n",
    "    target_unpacked_dir = os.path.join(data_root, \"CV_unpacked\")\n",
    "\n",
    "    if os.path.exists(target_unpacked_dir):\n",
    "        print('Find existing folder {}'.format(target_unpacked_dir))\n",
    "    else:\n",
    "        print(\"Could not find Common Voice, Downloading corpus...\")\n",
    "\n",
    "        filename = wget.download(COMMON_VOICE_URL, data_root)\n",
    "        target_file = os.path.join(data_root, os.path.basename(filename))\n",
    "\n",
    "        os.makedirs(target_unpacked_dir, exist_ok=True)\n",
    "        print(\"Unpacking corpus to {} ...\".format(target_unpacked_dir))\n",
    "        tar = tarfile.open(target_file, encoding=\"utf8\")\n",
    "        tar.extractall(target_unpacked_dir)\n",
    "        tar.close()\n",
    "\n",
    "    folder_path = os.path.join(target_unpacked_dir, args.version + f'/{args.language}/')\n",
    "\n",
    "    for csv_file in args.files_to_process:\n",
    "        data = process_files(\n",
    "            csv_file=os.path.join(folder_path, csv_file),\n",
    "            data_root=os.path.join(data_root, os.path.splitext(csv_file)[0]),\n",
    "            num_workers=args.num_workers\n",
    "        )\n",
    "        logging.info('Creating manifests...')\n",
    "        create_manifest(\n",
    "            data=data,\n",
    "            output_name=f'commonvoice_{os.path.splitext(csv_file)[0]}_manifest.jsonl',\n",
    "            manifest_path=args.manifest_dir,\n",
    "        )\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}