main.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install pdfminer.six pytesseract opencv-python python-docx openai==0.28 sentence-transformers spacy skillNer beautifulsoup4"
      ],
      "metadata": {
        "id": "lEgpw2HAGNa3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8fed5bc8-9b3a-4498-b39c-07e616f49f0e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pdfminer.six in /usr/local/lib/python3.10/dist-packages (20231228)\n",
            "Requirement already satisfied: pytesseract in /usr/local/lib/python3.10/dist-packages (0.3.10)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.10/dist-packages (4.8.0.76)\n",
            "Requirement already satisfied: python-docx in /usr/local/lib/python3.10/dist-packages (1.1.0)\n",
            "Requirement already satisfied: openai==0.28 in /usr/local/lib/python3.10/dist-packages (0.28.0)\n",
            "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.10/dist-packages (2.5.1)\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.10/dist-packages (3.7.4)\n",
            "Requirement already satisfied: skillNer in /usr/local/lib/python3.10/dist-packages (1.0.3)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (4.12.3)\n",
            "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai==0.28) (2.31.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai==0.28) (4.66.2)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai==0.28) (3.9.3)\n",
            "Requirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six) (3.3.2)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six) (42.0.5)\n",
            "Requirement already satisfied: packaging>=21.3 in /usr/local/lib/python3.10/dist-packages (from pytesseract) (23.2)\n",
            "Requirement already satisfied: Pillow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from pytesseract) (9.4.0)\n",
            "Requirement already satisfied: numpy>=1.21.2 in /usr/local/lib/python3.10/dist-packages (from opencv-python) (1.25.2)\n",
            "Requirement already satisfied: lxml>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from python-docx) (4.9.4)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from python-docx) (4.10.0)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.32.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.38.1)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (2.1.0+cu121)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.11.4)\n",
            "Requirement already satisfied: huggingface-hub>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (0.20.3)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.3.0,>=8.2.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (8.2.3)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.0.10)\n",
            "Requirement already satisfied: weasel<0.4.0,>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (0.3.4)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (0.9.0)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy) (6.4.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.6.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy) (67.7.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.3.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from skillNer) (1.5.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from skillNer) (3.8.1)\n",
            "Requirement already satisfied: jellyfish in /usr/local/lib/python3.10/dist-packages (from skillNer) (1.0.3)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4) (2.5)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=36.0.0->pdfminer.six) (1.16.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (3.13.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (2023.6.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence-transformers) (6.0.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (2.16.3)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai==0.28) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai==0.28) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai==0.28) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy) (0.1.4)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (3.2.1)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers) (2.1.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence-transformers) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence-transformers) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.32.0->sentence-transformers) (0.4.2)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy) (8.1.7)\n",
            "Requirement already satisfied: cloudpathlib<0.17.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from weasel<0.4.0,>=0.1.0->spacy) (0.16.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.28) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.28) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.28) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.28) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.28) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai==0.28) (4.0.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy) (2.1.5)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->skillNer) (1.3.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->skillNer) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->skillNer) (2023.4)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers) (3.3.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six) (2.21)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->skillNer) (1.16.0)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.11.0->sentence-transformers) (1.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Imports"
      ],
      "metadata": {
        "id": "n7o4ZgkfH26i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import re\n",
        "import cv2\n",
        "from bs4 import BeautifulSoup\n",
        "import pytesseract\n",
        "from pdfminer.high_level import extract_text\n",
        "from docx import Document\n",
        "from dateutil import parser\n",
        "from datetime import datetime\n",
        "import openai\n",
        "import json\n",
        "import csv\n",
        "from sentence_transformers import SentenceTransformer,util\n",
        "import numpy as np\n",
        "import spacy\n",
        "from spacy.matcher import PhraseMatcher\n",
        "from skillNer.general_params import SKILL_DB\n",
        "from skillNer.skill_extractor_class import SkillExtractor\n",
        "from graphviz import Digraph\n",
        "!python -m spacy download en_core_web_lg\n",
        "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "PKrqLHU0H5xz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9b2b8712-2ea5-4a54-c055-8d7193167045"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting en-core-web-lg==3.7.1\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.7.1/en_core_web_lg-3.7.1-py3-none-any.whl (587.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m587.7/587.7 MB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy<3.8.0,>=3.7.2 in /usr/local/lib/python3.10/dist-packages (from en-core-web-lg==3.7.1) (3.7.4)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.3.0,>=8.2.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (8.2.3)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (2.0.10)\n",
            "Requirement already satisfied: weasel<0.4.0,>=0.1.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (0.3.4)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (0.9.0)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (4.66.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (2.6.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (23.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (3.3.0)\n",
            "Requirement already satisfied: numpy>=1.19.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (1.25.2)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (2.16.3)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.3.0,>=8.2.2->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (0.1.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (8.1.7)\n",
            "Requirement already satisfied: cloudpathlib<0.17.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from weasel<0.4.0,>=0.1.0->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (0.16.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<3.8.0,>=3.7.2->en-core-web-lg==3.7.1) (2.1.5)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_lg')\n",
            "\u001b[38;5;3m⚠ Restart to reload dependencies\u001b[0m\n",
            "If you are in a Jupyter or Colab notebook, you may need to restart Python in\n",
            "order to load all the package's dependencies. You can do this by selecting the\n",
            "'Restart kernel' or 'Restart runtime' option.\n",
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nlp = spacy.load(\"en_core_web_lg\")\n",
        "skill_extractor = SkillExtractor(nlp, SKILL_DB, PhraseMatcher)"
      ],
      "metadata": {
        "id": "F7_wW_2bIxBL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cc576454-b7a5-4f3d-efcb-30e41ba37a7a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "loading full_matcher ...\n",
            "loading abv_matcher ...\n",
            "loading full_uni_matcher ...\n",
            "loading low_form_matcher ...\n",
            "loading token_matcher ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LgpTqiV0-mAK"
      },
      "source": [
        "# Parsing Task:\n",
        "\n",
        "**Purpose of this section:**\n",
        "- This is the parsing task. We intend to take in resumes from user and return the text in the files.\n",
        "- Allowed Formats : pdf, docx, image('.jpg', '.jpeg', '.png', '.gif', '.tiff', '.tif'), html and .tex files\n",
        "- The `load_text_from_files` function iterates over files in a directory (`root`) and extracts text from each file based on its extension. It then stores the extracted text in a list (`extracted_content`) and returns the list.\n",
        "\n",
        "**Input Handling:**\n",
        "\n",
        "- The function takes one argument:`root`, which specifies the directory containing the files.\n",
        "\n",
        "**Processing Steps:**\n",
        "\n",
        "- For each file in the directory, the function checks its extension using `file_name.endswith()` and calls the appropriate extraction function (`extract_text_from_pdf`, `extract_text_from_docx`, `extract_text_from_img`, `extract_text_from_html`) based on the file type.\n",
        "- The extracted text is appended to the `extracted_content` list.\n",
        "\n",
        "**Output:**\n",
        "\n",
        "- The function returns a list (`extracted_content`) containing the extracted text from all files in the directory."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hom2uxiUP8xo"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "start_time = time.time()  # Recording the start time\n",
        "\n",
        "root = \"/content/drive/MyDrive/resume\"\n",
        "pdf_files = []\n",
        "docx_files = []\n",
        "img_files = []\n",
        "html_files = []\n",
        "\n",
        "for file_name in os.listdir(root):\n",
        "    file_path = os.path.join(root, file_name)\n",
        "    if file_name.endswith('.pdf'):\n",
        "        pdf_files.append(file_path)\n",
        "    elif file_name.endswith('.docx'):\n",
        "        docx_files.append(file_path)\n",
        "    elif file_name.endswith(('.jpg', '.jpeg', '.png', '.gif', '.tiff', '.tif')):\n",
        "        img_files.append(file_path)\n",
        "    elif file_name.endswith('.html'):\n",
        "        html_files.append(file_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rAnJe_13PD6w"
      },
      "outputs": [],
      "source": [
        "def extract_text_from_pdf(pdf_path):\n",
        "    text = extract_text(pdf_path)\n",
        "    return text\n",
        "\n",
        "def extract_text_from_img(img_path):\n",
        "    img = cv2.imread(img_path)\n",
        "    text = pytesseract.image_to_string(img)\n",
        "    return text\n",
        "\n",
        "def extract_text_from_docx(docx_path):\n",
        "    doc = Document(docx_path)\n",
        "    text = \"\"\n",
        "    for paragraph in doc.paragraphs:\n",
        "        text += paragraph.text + \"\\n\"\n",
        "    return text\n",
        "\n",
        "def extract_text_from_html(html_path):\n",
        "    with open(html_path, 'r', encoding='utf-8') as file:\n",
        "        soup = BeautifulSoup(file, 'html.parser')\n",
        "        text = soup.get_text(separator=' ')\n",
        "        return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fwNMVl_Fdv_N"
      },
      "outputs": [],
      "source": [
        "pdf_text = \"\"\"         HR ADMINISTRATOR/MARKETING ASSOCIATE\n",
        "\n",
        "HR ADMINISTRATOR       Summary     Dedicated Customer Service Manager with 15+ years of experience in Hospitality and Customer Service Management.   Respected builder and leader of customer-focused teams; strives to instill a shared, enthusiastic commitment to customer service.         Highlights         Focused on customer satisfaction  Team management  Marketing savvy  Conflict resolution techniques     Training and development  Skilled multi-tasker  Client relations specialist           Accomplishments      Missouri DOT Supervisor Training Certification  Certified by IHG in Customer Loyalty and Marketing by Segment   Hilton Worldwide General Manager Training Certification  Accomplished Trainer for cross server hospitality systems such as    Hilton OnQ  ,   Micros    Opera PMS   , Fidelio    OPERA    Reservation System (ORS) ,   Holidex    Completed courses and seminars in customer service, sales strategies, inventory control, loss prevention, safety, time management, leadership and performance assessment.        Experience      HR Administrator/Marketing Associate\n",
        "\n",
        "HR Administrator     Dec 2013   to   Current      Company Name   ï¼   City  ,   State     Helps to develop policies, directs and coordinates activities such as employment, compensation, labor relations, benefits, training, and employee services.  Prepares employee separation notices and related documentation  Keeps records of benefits plans participation such as insurance and pension plan, personnel transactions such as hires, promotions, transfers, performance reviews, and terminations, and employee statistics for government reporting.  Advises management in appropriate resolution of employee relations issues.  Administers benefits programs such as life, health, dental, insurance, pension plans, vacation, sick leave, leave of absence, and employee assistance.     Marketing Associate Â    Designed and created marketing collateral for sales meetings, trade shows and company executives.  Managed the in-house advertising program consisting of print and media collateral pieces.  Assisted in the complete design and launch of the company's website in 2 months.  Created an official company page on Facebook to facilitate interaction with customers.  Analyzed ratings and programming features of competitors to evaluate the effectiveness of marketing strategies.         Advanced Medical Claims Analyst     Mar 2012   to   Dec 2013      Company Name   ï¼   City  ,   State     Reviewed medical bills for the accuracy of the treatments, tests, and hospital stays prior to sanctioning the claims.  Trained to interpret the codes (ICD-9, CPT) and terminology commonly used in medical billing to fully understand the paperwork that is submitted by healthcare providers.  Required to have organizational and analytical skills as well as computer skills, knowledge of medical terminology and procedures, statistics, billing standards, data analysis and laws regarding medical billing.         Assistant General Manager     Jun 2010   to   Dec 2010      Company Name   ï¼   City  ,   State     Performed duties including but not limited to, budgeting and financial management, accounting, human resources, payroll and purchasing.  Established and maintained close working relationships with all departments of the hotel to ensure maximum operation, productivity, morale and guest service.  Handled daily operations and reported directly to the corporate office.  Hired and trained staff on overall objectives and goals with an emphasis on high customer service.  Marketing and Advertising, working on public relations with the media, government and local businesses and Chamber of Commerce.         Executive Support / Marketing Assistant     Jul 2007   to   Jun 2010      Company Name   ï¼   City  ,   State     Provided assistance to various department heads - Executive, Marketing, Customer Service, Human Resources.  Managed front-end operations to ensure friendly and efficient transactions.  Ensured the swift resolution of customer issues to preserve customer loyalty while complying with company policies.  Exemplified the second-to-none customer service delivery in all interactions with customers and potential clients.         Reservation & Front Office Manager     Jun 2004   to   Jul 2007      Company Name   ï¼   City  ,   State          Owner/ Partner     Dec 2001   to   May 2004      Company Name   ï¼   City  ,   State          Price Integrity Coordinator     Aug 1999   to   Dec 2001      Company Name   ï¼   City  ,   State          Education      N/A  ,   Business Administration   1999     Jefferson College   ï¼   City  ,   State       Business Administration  Marketing / Advertising         High School Diploma  ,   College Prep. studies   1998     Sainte Genevieve Senior High   ï¼   City  ,   State       Awarded American Shrubel Leadership Scholarship to Jefferson College         Skills     Accounting, ads, advertising, analytical skills, benefits, billing, budgeting, clients, Customer Service, data analysis, delivery, documentation, employee relations, financial management, government relations, Human Resources, insurance, labor relations, layout, Marketing, marketing collateral, medical billing, medical terminology, office, organizational, payroll, performance reviews, personnel, policies, posters, presentations, public relations, purchasing, reporting, statistics, website.    \"\"\"\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ib9QoG8IU8Ip"
      },
      "source": [
        "# FILE LOADING:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "da5tvuZmU7aF"
      },
      "outputs": [],
      "source": [
        "def load_text_from_files(root):\n",
        "    extracted_content = []\n",
        "    for file_name in os.listdir(root):\n",
        "        if file_name.endswith('.pdf'):\n",
        "            pdf_resume_path = os.path.join(root, file_name)\n",
        "            pdf_text = extract_text_from_pdf(pdf_resume_path)\n",
        "            extracted_content.append(pdf_text)\n",
        "        elif file_name.endswith('.docx'):\n",
        "            docx_resume_path = os.path.join(root, file_name)\n",
        "            docx_text = extract_text_from_docx(docx_resume_path)\n",
        "            extracted_content.append(docx_text)\n",
        "        elif file_name.endswith('.html'):\n",
        "            html_resume_path = os.path.join(root, file_name)\n",
        "            html_text = extract_text_from_html(html_resume_path)\n",
        "            extracted_content.append(html_text)\n",
        "\n",
        "    return extracted_content"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-nmw-Qc7-u22"
      },
      "source": [
        "# PREPROCESSING:\n",
        "\n",
        "**Purpose:**\n",
        "\n",
        "- The `remove_punctuations` function removes specified punctuation characters (period and comma) from a given line using regular expressions.\n",
        "- The `preprocess_document` function preprocesses a document by converting each line to lowercase, removing punctuation, splitting into words, removing empty strings and spaces, and rejoining the words into a line.\n",
        "\n",
        "**Input Handling:**\n",
        "\n",
        "- The `remove_punctuations` function takes a single argument `line`, representing a single line of text.\n",
        "- The `preprocess_document` function takes a list `document` as input, where each element of the list is a line of text.\n",
        "\n",
        "**Processing Steps:**\n",
        "\n",
        "- `remove_punctuations`: Uses the `re.sub` function to replace periods and commas with an empty string in the input line.\n",
        "- `preprocess_document`:\n",
        "  - Converts each line to lowercase.\n",
        "  - Removes punctuation using `remove_punctuations`.\n",
        "  - Splits each line into a list of words using the space character as a delimiter.\n",
        "  - Removes any empty strings or spaces from the list.\n",
        "  - Joins the words back into a line using a space as a separator.\n",
        "  - Appends the preprocessed line to the `preprocessed_document` list if it is not empty."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XxffflaUgEC2"
      },
      "outputs": [],
      "source": [
        "def remove_punctuations(line):\n",
        "    return re.sub(r'[\\.,]', '', line)  # Simplified regex pattern\n",
        "\n",
        "def preprocess_document(document):\n",
        "    document = [line.strip() for line in document.split(\"\\n\") if line.strip()]\n",
        "    preprocessed_document = []  # Initialize an empty list to hold preprocessed lines\n",
        "    for line in document:\n",
        "        line = line.lower()\n",
        "        line = remove_punctuations(line)\n",
        "\n",
        "        line = line.split(' ')\n",
        "        line = [word for word in line if word.strip()]  # Remove empty strings and spaces\n",
        "\n",
        "        if line:  # Check if the list is not empty\n",
        "            preprocessed_document.append(' '.join(line))\n",
        "    return preprocessed_document\n",
        "\n",
        "# Preprocess document\n",
        "# preprocessed_doc = preprocess_document([line.strip() for line in pdf_text.split(\"\\n\") if line.strip()])  # Improved condition in list comprehension\n",
        "# preprocessed_doc\n",
        "# print(type(preprocessed_doc))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2wD-3SZ5-0zd"
      },
      "source": [
        "# Section Classification and ordering based on profiles:\n",
        "\n",
        "**Function Purpose:**\n",
        "\n",
        "- The `identify_sections` function categorizes lines of text from a document into different sections based on predefined rule based matching.\n",
        "\n",
        "**Benefit:**\n",
        "- The simple based matching outperformed in the results based on qualitative matching than complex techniques we tried such as creating embeddings and comparision and use of LLM. Plus it's less compute heavy\n",
        "\n",
        "**Input Handling:**\n",
        "\n",
        "- The function takes a list `document` as input, where each element of the list is a preprocessed line of text from the document.\n",
        "\n",
        "**Processing Steps:**\n",
        "\n",
        "- The function first initializes a dictionary `sections` with section names as keys and empty lists as values.\n",
        "- It then iterates over each line in the document.\n",
        "- For each line, it attempts to find a match with any of the predefined section synonyms using a regular expression pattern (`regex_pattern`).\n",
        "- If a match is found, the line is added to the corresponding section list in the `sections` dictionary.\n",
        "- If no match is found, the line is added to the current section if a section header has been previously identified.\n",
        "\n",
        "**Output:**\n",
        "\n",
        "- The function returns a dictionary `sections` containing the categorized lines of text for each section.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UqrAw52w0dRT"
      },
      "outputs": [],
      "source": [
        "def identify_sections(document):\n",
        "    # Dictionary of sections with their synonyms\n",
        "    sections_synonyms = {\n",
        "        'EDUCATION': ['EDUCATION'],\n",
        "        'EXPERIENCE': ['EXPERIENCE', 'WORK EXPERIENCE', 'PROFESSIONAL EXPERIENCES', 'CAREER EXPERIENCES'],\n",
        "        'PROJECTS': ['PROJECTS', \"PROJECT\"],\n",
        "        'SKILLS': ['SKILLS', 'TECHNICAL SKILLS', 'PROFESSIONAL SKILLS', 'CORE COMPETENCIES'],\n",
        "        'SUMMARY OVERVIEW': ['SUMMARY OVERVIEW', 'SUMMARY', 'OVERVIEW', 'PROFESSIONAL SUMMARY'],\n",
        "        'EXTRACURRICULAR': ['EXTRACURRICULAR', 'EXTRACURRICULAR ACTIVITIES'],\n",
        "        'ACHIEVEMENTS': ['ACHIEVEMENTS', 'AWARDS','AWARD', 'ACHIEVEMENT'],\n",
        "        'PUBLICATIONS': ['PUBLICATIONS', 'RESEARCH', 'RESEARCH WORK'],\n",
        "        'RESPONSIBILITIES': ['RESPONSIBILITIES', 'POR', 'POSITION OF RESPONSIBILITY']\n",
        "    }\n",
        "\n",
        "    sections = {key: [] for key in sections_synonyms.keys()}\n",
        "\n",
        "    current_section = None\n",
        "\n",
        "    regex_pattern = r'\\b(' + '|'.join([f\"({'|'.join(synonyms)})\" for synonyms in sections_synonyms.values()]) + r')\\b'\n",
        "    # regex_pattern = r'\\b(' + '|'.join([f\"({'|'.join([synonym + 's?' for synonym in synonyms])})\" for synonyms in sections_synonyms.values()]) + r')\\b'\n",
        "\n",
        "    for line in document:\n",
        "        header_match = re.search(regex_pattern, line, re.IGNORECASE)\n",
        "        if header_match:\n",
        "            for section, synonyms in sections_synonyms.items():\n",
        "                if header_match.group().upper() in map(str.upper, synonyms):\n",
        "                    current_section = section\n",
        "                    sections[current_section].append(line)\n",
        "                    break\n",
        "        else:\n",
        "            # If no header is matched, add the line to the current section\n",
        "            if current_section:\n",
        "                sections[current_section].append(line)\n",
        "\n",
        "    return sections"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kn44BNS7_uya"
      },
      "source": [
        "# Parsing Dates:\n",
        "**Function Purpose of get_date_time:**  \n",
        "   - The function `get_date_time` takes a date string as input, which can be in various formats, and returns a tuple `(START_DATE: datetime, END_DATE: datetime)`.\n",
        "\n",
        "**Input Handling:**  \n",
        "   - If the input string contains `\" - \"` or `\" to \"` (indicating a date range), it splits the string into `date_parts`.\n",
        "   - If the input string does not contain a date range delimiter, it considers the entire input as a single date part.\n",
        "\n",
        "**Parsing Start Date:**  \n",
        "   - It tries to parse the first part of the `date_parts` list (representing the start date) using `parser.parse` from the `dateutil` library.\n",
        "\n",
        "**Parsing End Date:**  \n",
        "   - If `date_parts` has more than one element (indicating an end date is present), it checks if the end date is \"present\". If it is, it sets the `end_date` to the current date and time using `datetime.now()`.\n",
        "   - Otherwise, it tries to parse the second part of `date_parts` (representing the end date) using `parser.parse`.\n",
        "   - If parsing fails, it returns `(None, None)`.\n",
        "\n",
        "**Formatting Dates:**  \n",
        "   - It converts the `start_date` and `end_date` to ISO format strings (`YYYY-MM-DDTHH:MM:SS`) if they are not `None`.\n",
        "\n",
        "**Returning Dates:**  \n",
        "   - It returns a tuple `(start_date, end_date)` where each date is either a valid ISO format string or `None` if parsing failed.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E_I5YEbpYfcD"
      },
      "outputs": [],
      "source": [
        "def get_date_time(str_input):\n",
        "  \"\"\" TAKES DATE STRINGS LIKE MAR-23 - APR 26 (and other various formats) AND RETURNS (START_DATE: datetime, END_DATE: datetime) \"\"\"\n",
        "  if \" - \" in str_input:\n",
        "      date_parts = str_input.split(\" - \")\n",
        "  elif \" to \" in str_input:\n",
        "      date_parts = str_input.split(\" to \")\n",
        "  else:\n",
        "      date_parts = [str_input]\n",
        "  try:\n",
        "      start_date = parser.parse(date_parts[0], default=datetime.min, dayfirst=True)\n",
        "  except ValueError:\n",
        "      return None, None\n",
        "\n",
        "  if len(date_parts) > 1:\n",
        "      if date_parts[1].strip().lower() == \"present\":\n",
        "          end_date = datetime.now()\n",
        "      else:\n",
        "          try:\n",
        "              end_date = parser.parse(date_parts[1], default=datetime.max, dayfirst=True)\n",
        "          except ValueError:\n",
        "              return None, None\n",
        "  else:\n",
        "      end_date = None\n",
        "\n",
        "  start_date = start_date.isoformat() if start_date is not None else None\n",
        "  end_date = end_date.isoformat() if end_date is not None else None\n",
        "  return (start_date, end_date)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Function Purpose generate_response - LLM api based part:**\n",
        "\n",
        "- The `generate_response` function generates a response to a system prompt followed by a user prompt using the OpenAI API.\n",
        "\n",
        "**Input Handling:**\n",
        "\n",
        "- The function takes three arguments: `system_prompt` (the prompt for the system), `user_prompt` (the prompt for the user), and `max_tokens` (maximum number of tokens in the response).\n",
        "\n",
        "**Processing Steps:**\n",
        "\n",
        "- The function uses the OpenAI API to create a completion for the given prompts (`system_prompt` and `user_prompt`) using the GPT-3.5-turbo-instruct engine.\n",
        "- The `prompt` parameter in the API call is formatted to include the system and user prompts.\n",
        "- The `max_tokens`, `temperature`, and `stop` parameters control the length and creativity of the response.\n",
        "\n",
        "**Output:**\n",
        "\n",
        "- The function returns the generated response as a string."
      ],
      "metadata": {
        "id": "iEBz9VkEMosw"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6kgciyFvIMI2"
      },
      "outputs": [],
      "source": [
        "openai.api_key =\"\"\n",
        "\n",
        "def generate_response(system_prompt, user_prompt, max_tokens=2048):\n",
        "    response = openai.Completion.create(\n",
        "        engine=\"gpt-3.5-turbo-instruct\",\n",
        "        prompt=f\"System:{system_prompt}\\nUser: {user_prompt}\\n\",\n",
        "        max_tokens=max_tokens,\n",
        "        temperature=0.7,\n",
        "        stop=\"\\nUser:\",\n",
        "    )\n",
        "    return response.choices[0].text.strip()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sSjkBHDFKWLJ"
      },
      "outputs": [],
      "source": [
        "education_system_prompt = \"\"\"\n",
        "You are going to write a JSON education section for an applicant applying for job posts.\n",
        "\n",
        "Now consider the following interface for the JSON schema:\n",
        "[{\n",
        "    \"subject_name\": string,\n",
        "    \"degree_type\": string,\n",
        "    \"institute_name\": string,\n",
        "    \"duration\": string | null,\n",
        "    \"description\": [string] | [],\n",
        "    \"location\": string | null,\n",
        "}]\n",
        "\n",
        "STRICTLY NOTE:\n",
        "- Strictly return JSON and do not use SINGLE QUOTES, only DOUBLE QUOTES for keys\n",
        "- Be truthful and objective to the skills listed in the CV\n",
        "- Be specific\n",
        "- Fix spelling and grammar errors\n",
        "- Give null if information is not given\n",
        "- Do not add or delete any extra information not present in the resume\n",
        "- while returning arrays make sure to not add an extra comma (',') at the end\n",
        "\n",
        "Write the education section according to the schema. In the response, include only the JSON.\n",
        "\"\"\"\n",
        "\n",
        "projects_system_prompt = \"\"\"\n",
        "You are going to write a JSON resume section for an applicant applying for job posts.\n",
        "\n",
        "Now consider the following interface for the JSON schema:\n",
        "\n",
        "[{\n",
        "    name: string,\n",
        "    description: string,\n",
        "    keywords: [string],\n",
        "    url: string,\n",
        "}]\n",
        "\n",
        "STRICTLY NOTE:\n",
        "- Strictly return JSON and do not use SINGLE QUOTES, only DOUBLE QUOTES for keys\n",
        "- Be truthful and objective to the skills listed in the CV\n",
        "- Be specific\n",
        "- Fix spelling and grammar errors\n",
        "- Give null if information is not present/given\n",
        "- Do not add or delete any extra information not present in the resume\n",
        "- while returning arrays make sure to not add an extra comma (',') at the end\n",
        "\n",
        "Write the projects section according to the Projects schema. Include all projects, but only the ones present in the CV. In the response, include only the JSON.\n",
        "\"\"\"\n",
        "\n",
        "skills_system_prompt = \"\"\"\n",
        "You are going to write a JSON resume section for an applicant applying for job posts.\n",
        "\n",
        "Now consider the following for the JSON schema:\n",
        "type HardSkills\n",
        "type SoftSkills\n",
        "type OtherSkills\n",
        "\n",
        "[{\n",
        "    \"name\": HardSkills | SoftSkills | OtherSkills,\n",
        "    \"keywords\": [string],\n",
        "}]\n",
        "\n",
        "STRICTLY NOTE:\n",
        "- Strictly return JSON and do not use SINGLE QUOTES, only DOUBLE QUOTES for keys\n",
        "- Be truthful and objective to the skills listed in the CV\n",
        "- Be specific\n",
        "- Fix spelling and grammar errors\n",
        "- Give null if information is not present/given\n",
        "- Do not add or delete any extra information not present in the resume\n",
        "- while returning arrays make sure to not add an extra comma (',') at the end\n",
        "\n",
        "Write the skills section according to the Skills schema. In the response, include only the JSON.\n",
        "\"\"\"\n",
        "\n",
        "\n",
        "work_experience_system_prompt = \"\"\"\n",
        "You are going to write a JSON resume section for an applicant applying for job posts.\n",
        "\n",
        "Now consider the following Interface for the JSON schema:\n",
        "[{\n",
        "    \"title\": string,\n",
        "    \"company_name\": string | \"N/A\",\n",
        "    \"duration\": string,\n",
        "    \"descriptions\": [string],\n",
        "}]\n",
        "\n",
        "STRICTLY NOTE:\n",
        "- Strictly return JSON and do not use SINGLE QUOTES, only DOUBLE QUOTES for keys\n",
        "- Be truthful and objective to the skills listed in the CV\n",
        "- Be specific\n",
        "- Fix spelling and grammar errors\n",
        "- Give null if information is not present/given\n",
        "- Do not add or delete any extra information not present in the resume\n",
        "- while returning arrays make sure to not add an extra comma (',') at the end\n",
        "\n",
        "Write a work section for the candidate according to the Work schema. Include only the work experience and not the project experience. No need to write the WORK keyword, just return the list.\n",
        "\"\"\"\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "- **Function Purpose:**\n",
        "  - The functions get_formatted_education_section, get_formatted_experience_section, and get_formatted_projects_section use a large language model (LLM) to fetch rich descriptions for sections.\n",
        "\n",
        "- **Input Handling:**\n",
        "  - The function takes a single argument `section`, which represents the input text section containing education information.\n",
        "\n",
        "- **Processing Steps:**\n",
        "  - The function generates a user prompt and uses it to generate a response using the `generate_response` function (assumed to utilize the LLM).\n",
        "  - The response is converted from JSON format to a Python dictionary (`structure`).\n",
        "  - The function then adds `start_date` and `end_date` to each education entry by calling the `get_date_time` function with the duration of each education entry.\n",
        "\n",
        "- **Output:**\n",
        "  - The function returns a structured JSON/dictionary containing education information with added `start_date` and `end_date`."
      ],
      "metadata": {
        "id": "uBwn1KKUMz-_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NZti_dQi1Dj2"
      },
      "outputs": [],
      "source": [
        "# GET STRUCTURED FORMAT FOR EACH SUB SECTION:\n",
        "# INITIAL AIM BEING 1) EDUCATION 2) WORK EXPERIENCE\n",
        "def get_formatted_education_section(section):\n",
        "  \"\"\" THIS SECTION USES LLM TO CONVERT INPUT SECTION TEXT TO STRUCTURED JSON/DICTIONARY \"\"\"\n",
        "  \"\"\"\n",
        "  [{\n",
        "    subject_name: string\n",
        "    degree_type: string\n",
        "    institute_name: string\n",
        "    duration: string\n",
        "    location: string\n",
        "    descriptions: [string]\n",
        "    start_date: ISODate\n",
        "    end_date: ISODate\n",
        "  }]\n",
        "  \"\"\"\n",
        "  education_user_prompt = f\"\"\"Apply this on the following {section} \"\"\"\n",
        "  education_response = generate_response(education_system_prompt, education_user_prompt)\n",
        "\n",
        "  structure = json.loads(education_response)\n",
        "  # structure = structure[\"education\"]\n",
        "  # print(structure)\n",
        "  # add start_date & end_date\n",
        "  for edu in structure:\n",
        "    print(edu[\"duration\"])\n",
        "    print(get_date_time(edu[\"duration\"]))\n",
        "    edu[\"start_date\"], edu[\"end_date\"] = get_date_time(edu[\"duration\"])\n",
        "\n",
        "  return structure\n",
        "\n",
        "def get_formatted_experience_section(section):\n",
        "  \"\"\" THIS SECTION USES LLM TO CONVERT INPUT SECTION TEXT TO STRUCTURED JSON/DICTIONARY \"\"\"\n",
        "  \"\"\"\n",
        "  [{\n",
        "    title: string\n",
        "    company_name: string\n",
        "    duration: string\n",
        "    descriptions: [string]\n",
        "    start_date: ISODate\n",
        "    end_date: ISODate\n",
        "    location: string\n",
        "\n",
        "    skills: {full_match: [string], predicted: [string]}\n",
        "    o_net_std: {code: String, title: String, description: String}\n",
        "  }]\n",
        "  \"\"\"\n",
        "  work_experience_user_prompt = f\"\"\"Apply this on the following {section} \"\"\"\n",
        "  work_experience_response = generate_response(work_experience_system_prompt, work_experience_user_prompt)\n",
        "  structure = json.loads(work_experience_response)\n",
        "\n",
        "  # structure = structure[\"experience\"]\n",
        "\n",
        "  for job in structure:\n",
        "    job[\"start_date\"], job[\"end_date\"] = get_date_time(job[\"duration\"])\n",
        "\n",
        "  return structure\n",
        "\n",
        "\n",
        "def get_formatted_project_section(section):\n",
        "  \"\"\" THIS SECTION USES LLM TO CONVERT INPUT SECTION TEXT TO STRUCTURED JSON/DICTIONARY \"\"\"\n",
        "  \"\"\"\n",
        "  [{\n",
        "    name: string;\n",
        "    description: string;\n",
        "    keywords: string[];\n",
        "    url: string;\n",
        "  }]\n",
        "  \"\"\"\n",
        "\n",
        "  project_section_user_prompt = f\"\"\"Apply this on the following {section} \"\"\"\n",
        "  project_section_response = generate_response(projects_system_prompt, project_section_user_prompt)\n",
        "\n",
        "  # print(project_section_response)\n",
        "  structure = json.loads(project_section_response)\n",
        "\n",
        "  return structure\n",
        "\n",
        "\n",
        "def get_formatted_skills_section(section):\n",
        "  \"\"\" THIS SECTION USES LLM TO CONVERT INPUT SECTION TEXT TO STRUCTURED JSON/DICTIONARY \"\"\"\n",
        "  \"\"\"\n",
        "  [{\n",
        "    name: HardSkills | SoftSkills | OtherSkills;\n",
        "    keywords: string[];\n",
        "  }]\n",
        "  \"\"\"\n",
        "  skill_section_user_prompt = f\"\"\"Apply this on the following {section} \"\"\"\n",
        "  skill_section_response = generate_response(skills_system_prompt, skill_section_user_prompt)\n",
        "\n",
        "  # print(skill_section_response)\n",
        "  structure = json.loads(skill_section_response)\n",
        "\n",
        "  return structure\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h8UraFwzNFkV"
      },
      "source": [
        "# APPLYING O-NET STANDARDIZATION TO JOB TITLES:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YRhievenPrs3"
      },
      "outputs": [],
      "source": [
        "def clean_string(string):\n",
        "  return re.sub(r'[^a-zA-Z0-9\\s]', '', string)\n",
        "\n",
        "def form_description_string(descriptions):\n",
        "  return \" \".join(descriptions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QKysfI3FXeP3"
      },
      "outputs": [],
      "source": [
        "def load_o_net_job_data(csv_file_path):\n",
        "  \"\"\" O-NET JOB STD DATA IS LOADED FROM CSV AND RETURNED AS TUPLE OF LISTS \"\"\"\n",
        "  codes = []\n",
        "  titles = []\n",
        "  descriptions = []\n",
        "\n",
        "  with open(csv_file_path, newline='', encoding='utf-8') as csvfile:\n",
        "    reader = csv.reader(csvfile)\n",
        "    next(reader)\n",
        "    for row in reader:\n",
        "      code = row[0]\n",
        "      title = row[1]\n",
        "      description = row[2]\n",
        "\n",
        "      descriptions.append(description)\n",
        "      titles.append(title)\n",
        "      codes.append(code)\n",
        "\n",
        "  return codes, titles, descriptions\n",
        "\n",
        "def prepare_o_net_job_description_embeddings(descriptions):\n",
        "  return model.encode(descriptions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oM3ZxXfedxlM"
      },
      "outputs": [],
      "source": [
        "# TODO: ENTER PATH HERE\n",
        "onet_job_std_csv_file_path = \"/content/drive/MyDrive/resume/2019_Occupations.csv\"\n",
        "\n",
        "o_net_job_codes, o_net_job_titles, o_net_job_descriptions = load_o_net_job_data(onet_job_std_csv_file_path)\n",
        "o_net_job_description_embeddings = prepare_o_net_job_description_embeddings(o_net_job_descriptions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eA4So5CHPFB5"
      },
      "outputs": [],
      "source": [
        "def get_o_net_std_job_format(job):\n",
        "  \"\"\"\n",
        "  ACCEPTS JOB STRUCTURE\n",
        "  JOB {\n",
        "    title: string\n",
        "    descriptions: [string],\n",
        "    ...\n",
        "    + adds a new field -> onet_job_std: (code:string, title:string, description: string)\n",
        "  }\n",
        "  \"\"\"\n",
        "\n",
        "  description_string = job[\"title\"] + \" \" + form_description_string(job[\"descriptions\"])\n",
        "  clean_description_string = clean_string(description_string)\n",
        "\n",
        "  # job decription embeddings\n",
        "  job_description_embeddings = model.encode(clean_description_string)\n",
        "\n",
        "  # apply cosine similarity\n",
        "  cos_sim = util.cos_sim(job_description_embeddings, o_net_job_description_embeddings)\n",
        "\n",
        "  max_sim_index = np.argmax(cos_sim, axis=1) # pick indexes of max\n",
        "  max_sim_index = max_sim_index[0]\n",
        "\n",
        "  code, title, description = o_net_job_codes[max_sim_index], o_net_job_titles[max_sim_index], o_net_job_descriptions[max_sim_index]\n",
        "  return {\"code\": code, \"title\": title, \"description\": description}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "50FInbOKrRkD"
      },
      "source": [
        "# SKILL EXTRACTION:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qmO9bPfBrmmw"
      },
      "outputs": [],
      "source": [
        "def extract_skills_using_descriptions(descriptions):\n",
        "  \"\"\" GET LIST OF SKILLS FROM DESCRIPTION SECTION OF A WORK EXPERIENCE \"\"\"\n",
        "\n",
        "  description_string = form_description_string(descriptions)\n",
        "  clean_description_string = clean_string(description_string)\n",
        "\n",
        "  annotations = skill_extractor.annotate(clean_description_string)\n",
        "  full_match_skills = list(map(lambda match_result: match_result[\"doc_node_value\"] , annotations[\"results\"][\"full_matches\"]))\n",
        "\n",
        "  # based on n-gram score, if score > 0.8\n",
        "  predicted_skills = [ match_result[\"doc_node_value\"] for match_result in annotations[\"results\"][\"ngram_scored\"] if match_result[\"score\"] > 0.8]\n",
        "\n",
        "  skills = {\"full_match\": full_match_skills, \"predicted\": predicted_skills }\n",
        "  return skills"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vBDWM6YjfQaV"
      },
      "source": [
        "# MAIN EXECUTION:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "9akabf5WfPNl",
        "outputId": "f27ba1d8-93c8-4c4d-9776-5db26f6c4a89"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sept 2021 - May 2023\n",
            "('2021-09-01T00:00:00', '2023-05-31T23:59:59.999999')\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "{'EDUCATION': [{'subject_name': 'Information Systems',\n",
              "   'degree_type': 'Master of Science',\n",
              "   'institute_name': 'New York University',\n",
              "   'duration': 'Sept 2021 - May 2023',\n",
              "   'description': ['Courses: Dealing with Data, Data Science for Business, Technical Database Management Systems, Real Time Big Data Analytics, Cloud Computing'],\n",
              "   'location': 'New York, NY',\n",
              "   'start_date': '2021-09-01T00:00:00',\n",
              "   'end_date': '2023-05-31T23:59:59.999999'}],\n",
              " 'EXPERIENCE': [{'title': 'Financial Systems and Data Intern',\n",
              "   'company_name': 'Clark Associates Inc',\n",
              "   'duration': 'May 2022 - Aug 2022',\n",
              "   'descriptions': ['Assisted in the management and organization of financial data for the company',\n",
              "    'Collaborated with team members to analyze financial trends and create reports',\n",
              "    'Used various software and tools to collect, clean, and analyze data for financial purposes',\n",
              "    'Presented findings and recommendations to senior management for decision making'],\n",
              "   'start_date': '2022-05-01T00:00:00',\n",
              "   'end_date': '2022-08-31T23:59:59.999999',\n",
              "   'o_net_std': {'code': '13-2011.00',\n",
              "    'title': 'Accountants and Auditors',\n",
              "    'description': 'Examine, analyze, and interpret accounting records to prepare financial statements, give advice, or audit and evaluate statements prepared by others. Install or advise on systems of recording costs or other financial and budgetary data.'},\n",
              "   'skills': {'full_match': ['financial datum'],\n",
              "    'predicted': ['management',\n",
              "     'collaborated',\n",
              "     'management',\n",
              "     'decision making']}}],\n",
              " 'PROJECTS': [{'name': 'Onsite Supply Chain Analytics Project - Doha, Qatar',\n",
              "   'description': 'Implemented a heuristic scoring model for approaching and on-boarding prospective customers that boosted conversions by 55%. Built 31 cross-sell/up-sell rule-based configuration for product recommendations with prescriptive analytics. Engaged with users in testing, release management, and operations to ensure quality of code development, deployment, and post-production support.',\n",
              "   'keywords': ['Tableau',\n",
              "    'MySQL',\n",
              "    'Heuristic Scoring',\n",
              "    'Cross-Sell/Up-Sell',\n",
              "    'Prescriptive Analytics',\n",
              "    'Release Management',\n",
              "    'Operations'],\n",
              "   'url': None},\n",
              "  {'name': 'Associate Software Engineer - Data Science & Quants Lab',\n",
              "   'description': 'Determined sales forecasting for 250+ retainer stores of the biggest conglomerate in UAE using Holt-Winters algorithm. Analysed data to develop a de-duplication algorithm model for grouping similar occurring brand names together. Segmented 5 million+ customers using RFM analysis to help the bank associates determine in advance the nature of future business operations and marketing strategies. Performed market basket analysis based on frequent purchasing patterns of customers.',\n",
              "   'keywords': ['Sales Forecasting',\n",
              "    'Holt-Winters Algorithm',\n",
              "    'De-Duplication',\n",
              "    'RFM Analysis',\n",
              "    'Market Basket Analysis'],\n",
              "   'url': None}],\n",
              " 'SKILLS': ['technical skills',\n",
              "  'databases & tools : mysql ssms ms access postgresql pl/sql mongodb',\n",
              "  'data analysis & visualization tools : python r sql ms excel tableau power bi',\n",
              "  'cloud platforms & big data technologies : aws (redshift ec2 s3 emr sagemaker dynamodb table) azure',\n",
              "  '(storage iot virtual machines) gcp (bigtable datalab) apache hadoop hive'],\n",
              " 'SUMMARY OVERVIEW': ['• extracted data from varied data sources to automate summary income financial reports(monthly quarterly and',\n",
              "  'yearly) for the various subsidiaries of clark associates inc that reduced manual entry time by 70%',\n",
              "  '• collaborated with cross-functional teams to enhance and optimize existing power bi reports that helped stakeholders',\n",
              "  'analyze the financial performance of their business',\n",
              "  'dacapo brokerage india private limited',\n",
              "  'senior analyst',\n",
              "  'mumbaiindia',\n",
              "  'aug 2020 - jul 2021',\n",
              "  'mumbaiindia',\n",
              "  'oct 2019 - jul 2020',\n",
              "  '• created visualization of daily top gainer/loser index stocks relative sector performance and analyzed technical trends',\n",
              "  'such as moving average of stock prices to help clients make informed investment decisions',\n",
              "  'crisil limited an s&p global company',\n",
              "  'senior associate - data science & quants lab',\n",
              "  '• lead a team of 2 interns to develop reports with functional specifications like kpi metrics revenue tracking budget',\n",
              "  'vs sales to understand opportunities in the sales funnel and improve operational effiency by 50% for the senior',\n",
              "  'management of crisil’s business development operations team'],\n",
              " 'EXTRACURRICULAR': [],\n",
              " 'ACHIEVEMENTS': ['leadership and achievements',\n",
              "  '• conferred quarterly award(service excellence award - q4 2019) at crisil limited',\n",
              "  '• received crisilite award for performance (clap) for the month of may 2019',\n",
              "  '• treasurer of computer society of india-vesit student chapter for the year 2016-2017',\n",
              "  '• executive head of sponsorship committee in the inter-collegiate technical festival'],\n",
              " 'PUBLICATIONS': [],\n",
              " 'RESPONSIBILITIES': []}"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "import os\n",
        "root = \"/content/drive/MyDrive/resume\"\n",
        "\n",
        "# EXTRACT CONTENT FROM FILES\n",
        "extracted_content = load_text_from_files(root)\n",
        "processed_documents = []\n",
        "\n",
        "preprocessed = preprocess_document(extracted_content[0])\n",
        "\n",
        "# PERFORM SEGMENTATION ON EXTRACTED CONTENT\n",
        "sections = identify_sections(preprocessed)\n",
        "\n",
        "# STRUCTURE SECTIONS\n",
        "if sections[\"EDUCATION\"]:\n",
        "  sections[\"EDUCATION\"] = get_formatted_education_section(sections[\"EDUCATION\"])\n",
        "\n",
        "if sections[\"EXPERIENCE\"]:\n",
        "  sections[\"EXPERIENCE\"] = get_formatted_experience_section(sections[\"EXPERIENCE\"])\n",
        "\n",
        "if sections[\"PROJECTS\"]:\n",
        "    sections[\"PROJECTS\"] = get_formatted_project_section(sections[\"PROJECTS\"])\n",
        "\n",
        "for job in sections[\"EXPERIENCE\"]:\n",
        "  job[\"o_net_std\"] = get_o_net_std_job_format(job)\n",
        "  job[\"skills\"] = extract_skills_using_descriptions(job[\"descriptions\"])\n",
        "\n",
        "display(sections)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8OnCC-VrWbfb"
      },
      "source": [
        "# Display Graph"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datetime import datetime\n",
        "from graphviz import Digraph\n",
        "\n",
        "# Sort education and experience data by start date\n",
        "sorted_education = sorted(data['EDUCATION'], key=lambda x: datetime.strptime(x['start_date'], \"%Y-%m-%dT%H:%M:%S\"))\n",
        "sorted_experience = sorted(data['EXPERIENCE'], key=lambda x: datetime.strptime(x['start_date'], \"%Y-%m-%dT%H:%M:%S\"))\n",
        "\n",
        "# Initialize Digraph\n",
        "dot = Digraph()\n",
        "\n",
        "# Set rank direction to top to bottom\n",
        "dot.attr(rankdir='RB')\n",
        "\n",
        "# Create main subgraph for education\n",
        "with dot.subgraph(name='cluster_education') as edu_subgraph:\n",
        "    # Add education heading\n",
        "    edu_subgraph.attr(label='Education', style='filled', fillcolor='lightgrey')\n",
        "\n",
        "    # Add nodes for education\n",
        "    for i, edu in enumerate(sorted_education):\n",
        "        # Calculate duration\n",
        "        start_date = datetime.strptime(edu['start_date'], \"%Y-%m-%dT%H:%M:%S\")\n",
        "        end_date = datetime.strptime(edu['end_date'], \"%Y-%m-%dT%H:%M:%S.%f\")\n",
        "        duration_in_months = (end_date.year - start_date.year) * 12 + (end_date.month - start_date.month)\n",
        "        years = duration_in_months // 12\n",
        "        months = duration_in_months % 12\n",
        "        duration_str = f\"{years} years, {months} months\"\n",
        "\n",
        "        # Format node label with university, duration, and descriptions\n",
        "        description_str = '\\n'.join(edu['description'])\n",
        "        node_label = f\"{edu['subject_name']} | {edu['institute_name']}\\n{duration_str}\\nDescriptions:\\n{description_str}\"\n",
        "\n",
        "        # Add node to education subgraph\n",
        "        edu_subgraph.node(f\"edu_node_{i}\", node_label, shape='box', style='filled', fillcolor='lightblue')\n",
        "\n",
        "        # Add arrows between nodes in sorted order\n",
        "        if i > 0:\n",
        "            edu_subgraph.edge(f\"edu_node_{i-1}\", f\"edu_node_{i}\")\n",
        "\n",
        "# Create second subgraph for experience\n",
        "with dot.subgraph(name='cluster_experience') as exp_subgraph:\n",
        "    # Add experience heading\n",
        "    exp_subgraph.attr(label='Experience', style='filled', fillcolor='lightgrey')\n",
        "\n",
        "    # Add nodes for experience\n",
        "    for i, exp in enumerate(sorted_experience):\n",
        "        # Calculate duration\n",
        "        start_date = datetime.strptime(exp['start_date'], \"%Y-%m-%dT%H:%M:%S\")\n",
        "        end_date = datetime.strptime(exp['end_date'], \"%Y-%m-%dT%H:%M:%S.%f\")\n",
        "        duration_in_months = (end_date.year - start_date.year) * 12 + (end_date.month - start_date.month)\n",
        "        years = duration_in_months // 12\n",
        "        months = duration_in_months % 12\n",
        "        duration_str = f\"{years} years, {months} months\"\n",
        "\n",
        "        # Format node label with work title, duration, and descriptions\n",
        "        description_str = '\\n'.join(exp['descriptions'])\n",
        "        node_label = f\"{exp['title']} | {exp['company_name']}\\n{duration_str}\\nDescriptions:\\n{description_str}\"\n",
        "\n",
        "        # Add node to experience subgraph\n",
        "        exp_subgraph.node(f\"exp_node_{i}\", node_label, shape='box', style='filled', fillcolor='lightblue')\n",
        "\n",
        "        # Add arrows between nodes in sorted order\n",
        "        if i > 0:\n",
        "            exp_subgraph.edge(f\"exp_node_{i-1}\", f\"exp_node_{i}\")\n",
        "\n",
        "# Render and display the graph\n",
        "dot.render('career_trajectory_sorted_with_dates_duration_arrows', format='png', cleanup=True)\n",
        "dot\n"
      ],
      "metadata": {
        "id": "XaR1vUiAu4cg",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        },
        "outputId": "59562227-af08-4458-e69e-db7a26eaf607"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.43.0 (0)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"1400pt\" height=\"176pt\"\n viewBox=\"0.00 0.00 1400.00 176.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 172)\">\n<title>%3</title>\n<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-172 1396,-172 1396,4 -4,4\"/>\n<g id=\"clust1\" class=\"cluster\">\n<title>cluster_education</title>\n<polygon fill=\"lightgrey\" stroke=\"black\" points=\"8,-30.5 8,-137.5 858,-137.5 858,-30.5 8,-30.5\"/>\n<text text-anchor=\"middle\" x=\"433\" y=\"-122.3\" font-family=\"Times,serif\" font-size=\"14.00\">Education</text>\n</g>\n<g id=\"clust2\" class=\"cluster\">\n<title>cluster_experience</title>\n<polygon fill=\"lightgrey\" stroke=\"black\" points=\"866,-8 866,-160 1384,-160 1384,-8 866,-8\"/>\n<text text-anchor=\"middle\" x=\"1125\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\">Experience</text>\n</g>\n<!-- edu_node_0 -->\n<g id=\"node1\" class=\"node\">\n<title>edu_node_0</title>\n<polygon fill=\"lightblue\" stroke=\"black\" points=\"849.5,-106.5 16.5,-106.5 16.5,-38.5 849.5,-38.5 849.5,-106.5\"/>\n<text text-anchor=\"middle\" x=\"433\" y=\"-91.3\" font-family=\"Times,serif\" font-size=\"14.00\">Information Systems | New York University</text>\n<text text-anchor=\"middle\" x=\"433\" y=\"-76.3\" font-family=\"Times,serif\" font-size=\"14.00\">1 years, 8 months</text>\n<text text-anchor=\"middle\" x=\"433\" y=\"-61.3\" font-family=\"Times,serif\" font-size=\"14.00\">Descriptions:</text>\n<text text-anchor=\"middle\" x=\"433\" y=\"-46.3\" font-family=\"Times,serif\" font-size=\"14.00\">Courses: Dealing with Data, Data Science for Business, Technical Database Management Systems, Real Time Big Data Analytics, Cloud Computing</text>\n</g>\n<!-- exp_node_0 -->\n<g id=\"node2\" class=\"node\">\n<title>exp_node_0</title>\n<polygon fill=\"lightblue\" stroke=\"black\" points=\"1376,-129 874,-129 874,-16 1376,-16 1376,-129\"/>\n<text text-anchor=\"middle\" x=\"1125\" y=\"-113.8\" font-family=\"Times,serif\" font-size=\"14.00\">Financial Systems and Data Intern | Clark Associates Inc</text>\n<text text-anchor=\"middle\" x=\"1125\" y=\"-98.8\" font-family=\"Times,serif\" font-size=\"14.00\">0 years, 3 months</text>\n<text text-anchor=\"middle\" x=\"1125\" y=\"-83.8\" font-family=\"Times,serif\" font-size=\"14.00\">Descriptions:</text>\n<text text-anchor=\"middle\" x=\"1125\" y=\"-68.8\" font-family=\"Times,serif\" font-size=\"14.00\">Assisted in the management and organization of financial data for the company</text>\n<text text-anchor=\"middle\" x=\"1125\" y=\"-53.8\" font-family=\"Times,serif\" font-size=\"14.00\">Collaborated with team members to analyze financial trends and create reports</text>\n<text text-anchor=\"middle\" x=\"1125\" y=\"-38.8\" font-family=\"Times,serif\" font-size=\"14.00\">Used various software and tools to collect, clean, and analyze data for financial purposes</text>\n<text text-anchor=\"middle\" x=\"1125\" y=\"-23.8\" font-family=\"Times,serif\" font-size=\"14.00\">Presented findings and recommendations to senior management for decision making</text>\n</g>\n</g>\n</svg>\n",
            "text/plain": [
              "<graphviz.graphs.Digraph at 0x7a0e97b4d120>"
            ]
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "end_time = time.time()\n",
        "total_runtime = end_time - start_time\n",
        "print(f\"Total run time: {total_runtime} seconds\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yEjm0Z1NDdUp",
        "outputId": "f5613afc-d06b-49c1-9b05-0e0ee9059bb5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total run time: 574.0707130432129 seconds\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}