2_Preprocessing_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2- Preprocessing data.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Malekkamoua/Medical-data-classifier/blob/main/2_Preprocessing_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4JzokFkCP4m7"
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "120sqw3-Q9_B"
      },
      "source": [
        "!unzip  \"/content/gdrive/MyDrive/PFA DATA/Clean_data/Copy of clean_data.zip\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HxmYU7mL9hx7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6db7ba1a-212d-4101-d8fc-fef4eddee413"
      },
      "source": [
        "pip install nltk==3.4.5"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting nltk==3.4.5\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f6/1d/d925cfb4f324ede997f6d47bea4d9babba51b49e87a767c170b77005889d/nltk-3.4.5.zip (1.5MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5MB 5.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk==3.4.5) (1.15.0)\n",
            "Building wheels for collected packages: nltk\n",
            "  Building wheel for nltk (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nltk: filename=nltk-3.4.5-cp37-none-any.whl size=1449908 sha256=70b9cdf627ea2beb85762eb2cadefcea911aa151599e9747403b1b0407390d39\n",
            "  Stored in directory: /root/.cache/pip/wheels/96/86/f6/68ab24c23f207c0077381a5e3904b2815136b879538a24b483\n",
            "Successfully built nltk\n",
            "Installing collected packages: nltk\n",
            "  Found existing installation: nltk 3.2.5\n",
            "    Uninstalling nltk-3.2.5:\n",
            "      Successfully uninstalled nltk-3.2.5\n",
            "Successfully installed nltk-3.4.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KzD4bKQo9kvl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8399e464-57e1-424d-ce96-5effc3564fcb"
      },
      "source": [
        "import nltk  \n",
        "\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('averaged_perceptron_tagger')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c6VIxOJBKnER"
      },
      "source": [
        "import re\n",
        "import string\n",
        "from nltk.corpus import stopwords \n",
        "from nltk.tokenize import word_tokenize \n",
        "from nltk.stem import WordNetLemmatizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EKCUOADGhSRw"
      },
      "source": [
        "#Displaying folders' names\n",
        "import os\n",
        "arr = os.listdir(\"/content/content/clean_data\")\n",
        "print(arr[0])\n",
        "print(arr[5])\n",
        "print(arr[4])\n",
        "print(arr[0:5]) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ArT9rXGTbajA"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "DNpWWD_woATa"
      },
      "source": [
        "import sys\n",
        "\n",
        "errors = 0\n",
        "\n",
        "for file in arr[40000:43000]:\n",
        "  try: \n",
        "\n",
        "    single_filename = file\n",
        "          \n",
        "    file1 = open(r'/content/content/clean_data/'+file, \"r\",encoding=None, errors=None, newline=None,closefd=True, opener=None)\n",
        "    content = file1.read()\n",
        "    filename = file1.name\n",
        " \n",
        "    #To Lower case\n",
        "    content = content.lower()\n",
        "\n",
        "    #Remove numbers\n",
        "    content = re.sub(r'\\d+', '', content)\n",
        " \n",
        "    #Removing punctuation\n",
        "    input_str= content.translate(str.maketrans('','',string.punctuation))\n",
        "    content = input_str.translate(str.maketrans('','',string.punctuation))\n",
        "\n",
        "    #removing white spaces\n",
        "    content = content.strip()\n",
        "\n",
        "    #Removing stop words\n",
        "    stop_words = set(stopwords.words('english')) \n",
        "    word_tokens = word_tokenize(content)\n",
        "            \n",
        "    content = [w for w in word_tokens if not w in stop_words] \n",
        " \n",
        "    #Lemmatization \n",
        "    wnl = WordNetLemmatizer()\n",
        "\n",
        "    lemmatized_string = ' '.join([wnl.lemmatize(words) for words in content])\n",
        " \n",
        "    # Save preprocessed file \n",
        "    title = '/content/content/preprocessed_data_part13/'+single_filename\n",
        "    file = open(title, \"w\") \n",
        "    file.write(lemmatized_string) \n",
        "    file.close() \n",
        "        \n",
        "  #encoding error blocking the process\n",
        "  except Exception  as error:\n",
        "    print('An exception occurred: {}'.format(error))\n",
        "    continue\n",
        "    \n",
        "print(\"All done with success :) \")\n",
        "        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "22z3d0vHkwMi"
      },
      "source": [
        "!zip -r /content/content/preprocessed_data_part13 /content/content/preprocessed_data_part13"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CW6-ni59k7RI"
      },
      "source": [
        "# mount it\n",
        "from google.colab import drive\n",
        "drive.mount(\"/content/gdrive\", force_remount=True)\n",
        "# copy it there\n",
        "!cp \"/content/content/preprocessed_data_part13.zip\" \"/content/gdrive/My Drive/PFA DATA/ready/preprocessed_data_part13.zip\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wW7jvA6wirZc"
      },
      "source": [
        "file1 = open(r'/content/content/clean_data/'+arr[1], \"r\",encoding=None, errors=None, newline=None,closefd=True, opener=None)\n",
        "content = file1.read()\n",
        "filename = file1.name\n",
        "\n",
        "print(content)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4rJjwkXX7p1B",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b08aeb62-afbf-4407-ee58-1be0217e0b60"
      },
      "source": [
        "content = \"We calculate the square of euclidean distances between centroids and data points. The data point is assigned to the kth cluster if square of euclidean distance between the data point and the centroid of kth cluster is minimum.\"\n",
        "#To Lower case\n",
        "content = content.lower()\n",
        "\n",
        "#Remove numbers\n",
        "content = re.sub(r'\\d+', '', content)\n",
        "\n",
        "#Removing punctuation\n",
        "input_str= content.translate(str.maketrans('','',string.punctuation))\n",
        "content = input_str.translate(str.maketrans('','',string.punctuation))\n",
        "\n",
        "#removing white spaces\n",
        "content = content.strip()\n",
        "\n",
        "#Removing stop words\n",
        "stop_words = set(stopwords.words('english')) \n",
        "word_tokens = word_tokenize(content)\n",
        "\n",
        "content = [w for w in word_tokens if not w in stop_words] \n",
        "   \n",
        "#Lemmatization \n",
        "wnl = WordNetLemmatizer()\n",
        "\n",
        "lemmatized_string = [wnl.lemmatize(words) for words in content]\n",
        "print(lemmatized_string)   \n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['calculate', 'square', 'euclidean', 'distance', 'centroid', 'data', 'point', 'data', 'point', 'assigned', 'kth', 'cluster', 'square', 'euclidean', 'distance', 'data', 'point', 'centroid', 'kth', 'cluster', 'minimum']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vALk82FYhGkA"
      },
      "source": [
        "#Javscript code to prevent colab from disconnecting\n",
        "function ClickConnect(){\n",
        "console.log(\"Working\");      \n",
        "document.querySelector(\"colab-toolbar-button\").click() \n",
        "}setInterval(ClickConnect,60000)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}