combine_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-19T02:25:28.856994Z",
     "start_time": "2024-09-19T02:25:28.838998Z"
    }
   },
   "outputs": [],
   "source": [
    "# Combine all data into one file\n",
    "import pandas as pd\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "def sanitize_sentence(sentence):\n",
    "    \"\"\"\n",
    "    Sanitize a sentence\n",
    "    :param sentence:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    sentence = re.sub(r'={2,}', '', sentence)\n",
    "    sentence = re.sub(r'\\[\\d+]', '', sentence)\n",
    "    sentence = re.sub(r'\\([^)]*\\)', '', sentence)\n",
    "    sentence = re.sub(r'\\s+\\.', '.', sentence)\n",
    "    sentence = re.sub(r'\\s+', ' ', sentence)\n",
    "    sentence = sentence.strip()\n",
    "    return sentence"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-09-19T02:25:28.872993Z",
     "start_time": "2024-09-19T02:25:28.858994Z"
    }
   },
   "id": "aebf6b1e5624b1e0",
   "execution_count": 41
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# Read all data\n",
    "# get all data from synthetic data\n",
    "data_path = 'datasets/printed/synthetic'\n",
    "data_files = os.listdir(data_path)\n",
    "\n",
    "# look for the file image_labels_dataset.csv, and get the path\n",
    "data_files = [os.path.join(data_path, file) for file in data_files]\n",
    "# change the characters \\\\ to /\n",
    "data_files = [file.replace('\\\\', '/') for file in data_files]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-09-19T02:25:28.887994Z",
     "start_time": "2024-09-19T02:25:28.874994Z"
    }
   },
   "id": "90c1d3167cb4d216",
   "execution_count": 42
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "def sanitize_sentences_and_save(__data_files):\n",
    "    # take only the files that have the extension sentences.csv\n",
    "    for file in __data_files:\n",
    "        if not os.path.isdir(file):\n",
    "            continue\n",
    "        dir_files = os.listdir(file)\n",
    "        for _file in dir_files:\n",
    "            if 'sentences.csv' in _file:\n",
    "                csv = pd.read_csv(str(os.path.join(file, _file)))\n",
    "                print(csv.shape[0])\n",
    "                # take the sentences and sanitize them\n",
    "                sentences = csv.sentence\n",
    "                \n",
    "                sentences = [sanitize_sentence(str(sentence)) for sentence in sentences]\n",
    "                # save the file in the same .csv file\n",
    "                csv.sentence = sentences\n",
    "                csv.to_csv(str(os.path.join(file, _file)), index=False)\n",
    "                print(\"File saved\")\n",
    "                    "
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-09-19T02:25:28.903996Z",
     "start_time": "2024-09-19T02:25:28.888994Z"
    }
   },
   "id": "42816a28b9f53237",
   "execution_count": 43
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "def combine_and_save_txt(__data_files):\n",
    "    with open('datasets/printed/synthetic/combined_data.txt', 'w', encoding='utf-8') as f:\n",
    "        print(f\"Creating file in {f.name}....\")\n",
    "        for file in __data_files:\n",
    "            if not os.path.isdir(file):\n",
    "                continue\n",
    "            dir_files = os.listdir(file)\n",
    "            for _file in dir_files:\n",
    "                if 'image_labels_dataset.csv' in _file:\n",
    "                    csv = pd.read_csv(str(os.path.join(file, _file)))\n",
    "                    print(csv.shape[0])\n",
    "                    # take the image path and add the prefix of the path\n",
    "                    image_paths = csv.image_path\n",
    "                    # take the last part of the path\n",
    "                    image_paths = [file.split('/')[-1] + '/' + image_path for image_path in image_paths]\n",
    "                    labels = csv.label\n",
    "                    # do not clean the labels here\n",
    "                    labels = [label for label in labels]\n",
    "                    #  create a txt file with the image path and the labels\n",
    "                    with open('datasets/printed/synthetic/combined_data.txt', 'a', encoding='utf-8') as g:\n",
    "                        for i in range(len(labels)):\n",
    "                            g.write(image_paths[i] + '  ' + labels[i] + '\\n')\n",
    "        print(\"File created\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-09-19T02:25:28.918998Z",
     "start_time": "2024-09-19T02:25:28.905999Z"
    }
   },
   "id": "1e35e3eed2c31380",
   "execution_count": 44
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120159\n",
      "File saved\n",
      "118308\n",
      "File saved\n"
     ]
    }
   ],
   "source": [
    "sanitize_sentences_and_save(data_files)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-09-19T02:25:31.962526Z",
     "start_time": "2024-09-19T02:25:28.919994Z"
    }
   },
   "id": "a339bd12a29e6a70",
   "execution_count": 45
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}