scraper_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'summarise_listing' from 'utils.cover_letter' (/Users/mahsongwei/Desktop/streamlit-cv-gen/AI-Cover-Letter-Generator/utils/cover_letter.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[24], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcover_letter\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m generate_cover_letter, generate_feedback, create_search_terms, summarise_listing\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'summarise_listing' from 'utils.cover_letter' (/Users/mahsongwei/Desktop/streamlit-cv-gen/AI-Cover-Letter-Generator/utils/cover_letter.py)"
     ]
    }
   ],
   "source": [
    "from utils.cover_letter import generate_cover_letter, generate_feedback, create_search_terms, summarise_listing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'summarise' from 'utils.cover_letter' (/Users/mahsongwei/Desktop/streamlit-cv-gen/AI-Cover-Letter-Generator/utils/cover_letter.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[21], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mstreamlit\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mst\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtext_extraction\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m extract_text_from_cv, extract_job_description_from_link, parse_file_contents\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcover_letter\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m generate_cover_letter, generate_feedback, create_search_terms, summarise\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mkey_validation\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m is_valid_key\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m#from utils.add_github_link import add_github_link\u001b[39;00m\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'summarise' from 'utils.cover_letter' (/Users/mahsongwei/Desktop/streamlit-cv-gen/AI-Cover-Letter-Generator/utils/cover_letter.py)"
     ]
    }
   ],
   "source": [
    "import streamlit as st\n",
    "from utils.text_extraction import extract_text_from_cv, extract_job_description_from_link, parse_file_contents\n",
    "from utils.cover_letter import generate_cover_letter, generate_feedback, create_search_terms, summarise_listing\n",
    "from utils.key_validation import is_valid_key\n",
    "#from utils.add_github_link import add_github_link\n",
    "\n",
    "\n",
    "import os\n",
    "import re\n",
    "import ast\n",
    "import magic\n",
    "from PyPDF2 import PdfReader\n",
    "import docx2txt\n",
    "import pytesseract\n",
    "from PIL import Image\n",
    "import json\n",
    "from groq import Groq\n",
    "# make sure you have .env file saved locally with your API keys\n",
    "from dotenv import load_dotenv\n",
    "import pandas as pd\n",
    "#import sqlite3\n",
    "from jobspy import scrape_jobs\n",
    "from st_aggrid import AgGrid, GridOptionsBuilder\n",
    "\n",
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import pprint\n",
    "from jobspy import scrape_jobs\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-14 11:38:52,826 - JobSpy - INFO - Indeed search page: 1\n",
      "2024-08-14 11:38:53,268 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-14 11:38:53,540 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-14 11:38:53,541 - JobSpy - INFO - Indeed finished scraping\n"
     ]
    }
   ],
   "source": [
    "a = scrape_jobs(\n",
    "      # site_name=[\"indeed\", \"linkedin\", \"glassdoor\"],\n",
    "      site_name=[\"indeed\"],\n",
    "      search_term=\"part-time childcare job\",\n",
    "      #location=\"Tanjung Malim\",\n",
    "      results_wanted=100,\n",
    "      hours_old=72,  # (only Linkedin/Indeed is hour specific, others round up to days old)\n",
    "      country_indeed=\"Malaysia\",  # only needed for indeed / glassdoor\n",
    "      # linkedin_fetch_description=True # get full description , direct job url , company industry and job level (seniority level) for linkedin (slower)\n",
    "      # proxies=[\"208.195.175.46:65095\", \"208.195.175.45:65095\", \"localhost\"],\n",
    "  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>site</th>\n",
       "      <th>job_url</th>\n",
       "      <th>job_url_direct</th>\n",
       "      <th>title</th>\n",
       "      <th>company</th>\n",
       "      <th>location</th>\n",
       "      <th>job_type</th>\n",
       "      <th>date_posted</th>\n",
       "      <th>salary_source</th>\n",
       "      <th>...</th>\n",
       "      <th>company_url</th>\n",
       "      <th>company_url_direct</th>\n",
       "      <th>company_addresses</th>\n",
       "      <th>company_num_employees</th>\n",
       "      <th>company_revenue</th>\n",
       "      <th>company_description</th>\n",
       "      <th>logo_photo_url</th>\n",
       "      <th>banner_photo_url</th>\n",
       "      <th>ceo_name</th>\n",
       "      <th>ceo_photo_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f183b6b015d360f4</td>\n",
       "      <td>indeed</td>\n",
       "      <td>https://malaysia.indeed.com/viewjob?jk=f183b6b...</td>\n",
       "      <td>http://malaysia.indeed.com/job/freelance-teach...</td>\n",
       "      <td>Freelance Teacher (Teach 2-3 Hrs Per Day Only)</td>\n",
       "      <td>Grownce Sdn Bhd</td>\n",
       "      <td>Alor Setar, M02, MY</td>\n",
       "      <td>parttime, temporary, contract</td>\n",
       "      <td>2024-08-12</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>https://malaysia.indeed.com/cmp/Grownce-Sdn-Bhd-1</td>\n",
       "      <td>http://www.grownce.com/</td>\n",
       "      <td>495A Jalan Persisiran Perling 1 Taman Perling ...</td>\n",
       "      <td>11 to 50</td>\n",
       "      <td>None</td>\n",
       "      <td></td>\n",
       "      <td>https://d2q79iu7y748jz.cloudfront.net/s/_squar...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id    site  \\\n",
       "0  f183b6b015d360f4  indeed   \n",
       "\n",
       "                                             job_url  \\\n",
       "0  https://malaysia.indeed.com/viewjob?jk=f183b6b...   \n",
       "\n",
       "                                      job_url_direct  \\\n",
       "0  http://malaysia.indeed.com/job/freelance-teach...   \n",
       "\n",
       "                                            title          company  \\\n",
       "0  Freelance Teacher (Teach 2-3 Hrs Per Day Only)  Grownce Sdn Bhd   \n",
       "\n",
       "              location                       job_type date_posted  \\\n",
       "0  Alor Setar, M02, MY  parttime, temporary, contract  2024-08-12   \n",
       "\n",
       "  salary_source  ...                                        company_url  \\\n",
       "0          None  ...  https://malaysia.indeed.com/cmp/Grownce-Sdn-Bhd-1   \n",
       "\n",
       "        company_url_direct                                  company_addresses  \\\n",
       "0  http://www.grownce.com/  495A Jalan Persisiran Perling 1 Taman Perling ...   \n",
       "\n",
       "  company_num_employees  company_revenue company_description  \\\n",
       "0              11 to 50             None                       \n",
       "\n",
       "                                      logo_photo_url banner_photo_url  \\\n",
       "0  https://d2q79iu7y748jz.cloudfront.net/s/_squar...             None   \n",
       "\n",
       "  ceo_name ceo_photo_url  \n",
       "0     None          None  \n",
       "\n",
       "[1 rows x 31 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    **Please watch below video:**\\n\\nTV2 Show\\n\\nh...\n",
       "Name: description, dtype: object"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a['description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#define search terms and locations\n",
    "#search_terms = [\"Accountant\", \"Engineer\", \"Software Developer\", \"Designer\", \"Guard\", \"Analyst\", \"Consultant\", \"Technician\", \"Nurse\", \"Gardener\"]\n",
    "#locations = [\"Tanjung Malim\", \"Kuala Lumpur\", \"Penang\", \"Johor Bahru\", \"Ipoh\", \"Kota Kinabalu\", \"Shah Alam\", \"Malacca\", \"Alor Setar\", \"Kuantan\"]\n",
    "\n",
    "search_terms = [\"Accountant\", \"Guard\", \"Teacher\", \"software developer\", \"Carer\", \"temporary\"]\n",
    "locations = [\"Kuala Lumpur\", \"Pahang\", \"Ipoh\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:20,707 - JobSpy - INFO - Indeed search page: 1\n",
      "2024-08-10 20:59:21,220 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:22,062 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:22,063 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:22,116 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Accountant, Location:Kuala Lumpur, Result Number:28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:23,200 - JobSpy - INFO - Indeed found no jobs on page: 1\n",
      "2024-08-10 20:59:23,201 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:23,211 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Accountant, Location:Pahang, Result Number:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:23,611 - JobSpy - INFO - Indeed found no jobs on page: 1\n",
      "2024-08-10 20:59:23,612 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:23,622 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Accountant, Location:Ipoh, Result Number:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:24,247 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:24,599 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:24,600 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:24,628 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Guard, Location:Kuala Lumpur, Result Number:10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:24,930 - JobSpy - INFO - Indeed found no jobs on page: 1\n",
      "2024-08-10 20:59:24,932 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:24,942 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Guard, Location:Pahang, Result Number:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:25,459 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:25,708 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:25,708 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:25,729 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Guard, Location:Ipoh, Result Number:1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:26,339 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:26,886 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:26,887 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:26,958 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Teacher, Location:Kuala Lumpur, Result Number:52\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:27,237 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:27,976 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:27,977 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:27,986 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Teacher, Location:Pahang, Result Number:1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:28,428 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:28,720 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:28,721 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:28,743 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Teacher, Location:Ipoh, Result Number:2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:29,445 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:29,939 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:29,939 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:30,020 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:software developer, Location:Kuala Lumpur, Result Number:89\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:30,470 - JobSpy - INFO - Indeed found no jobs on page: 1\n",
      "2024-08-10 20:59:30,471 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:30,480 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:software developer, Location:Pahang, Result Number:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:31,400 - JobSpy - INFO - Indeed found no jobs on page: 1\n",
      "2024-08-10 20:59:31,401 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:31,412 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:software developer, Location:Ipoh, Result Number:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:31,800 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:32,060 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:32,061 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:32,072 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Carer, Location:Kuala Lumpur, Result Number:4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:32,390 - JobSpy - INFO - Indeed found no jobs on page: 1\n",
      "2024-08-10 20:59:32,391 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:32,398 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Carer, Location:Pahang, Result Number:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:32,834 - JobSpy - INFO - Indeed found no jobs on page: 1\n",
      "2024-08-10 20:59:32,835 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:32,845 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:Carer, Location:Ipoh, Result Number:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:33,588 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:34,067 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:34,068 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:34,124 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:temporary, Location:Kuala Lumpur, Result Number:27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:34,673 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:34,971 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:34,972 - JobSpy - INFO - Indeed finished scraping\n",
      "2024-08-10 20:59:34,990 - JobSpy - INFO - Indeed search page: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:temporary, Location:Pahang, Result Number:1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-10 20:59:35,499 - JobSpy - INFO - Indeed search page: 2\n",
      "2024-08-10 20:59:35,902 - JobSpy - INFO - Indeed found no jobs on page: 2\n",
      "2024-08-10 20:59:35,903 - JobSpy - INFO - Indeed finished scraping\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search Term:temporary, Location:Ipoh, Result Number:3\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Initialize an empty DataFrame\n",
    "results_df = pd.DataFrame()\n",
    "\n",
    "# Loop over each search term and location\n",
    "for search_term in search_terms:\n",
    "    for location in locations:\n",
    "        try:\n",
    "            a = scrape_jobs(\n",
    "                # site_name=xxx,\n",
    "                site_name=[\"indeed\"],  \n",
    "                search_term=search_term,\n",
    "                location=location,\n",
    "                results_wanted=100,\n",
    "                hours_old=168,  \n",
    "                country_indeed=\"Malaysia\",\n",
    "            )\n",
    "            #print(f\"Search Term: {search_term}, Location: {location}\")\n",
    "            print (f\"Search Term:{search_term}, Location:{location}, Result Number:{len(a)}\")\n",
    "            #print(len(a))  # Adjust this based on how you want to handle/display the results\n",
    "            a['input_search_term'] = search_term\n",
    "            a['input_location'] = location\n",
    "            results_df = pd.concat([results_df, a], ignore_index=True)\n",
    "        except Exception as e:\n",
    "            print(f\"Error Details: {str(e)}\")\n",
    "            continue\n",
    "\n",
    "\n",
    "results_df.to_csv('scrapertest2_1008.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>site</th>\n",
       "      <th>job_url</th>\n",
       "      <th>job_url_direct</th>\n",
       "      <th>title</th>\n",
       "      <th>company</th>\n",
       "      <th>location</th>\n",
       "      <th>job_type</th>\n",
       "      <th>date_posted</th>\n",
       "      <th>salary_source</th>\n",
       "      <th>...</th>\n",
       "      <th>company_addresses</th>\n",
       "      <th>company_num_employees</th>\n",
       "      <th>company_revenue</th>\n",
       "      <th>company_description</th>\n",
       "      <th>logo_photo_url</th>\n",
       "      <th>banner_photo_url</th>\n",
       "      <th>ceo_name</th>\n",
       "      <th>ceo_photo_url</th>\n",
       "      <th>input_search_term</th>\n",
       "      <th>input_location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>22e518104ecb405b</td>\n",
       "      <td>indeed</td>\n",
       "      <td>https://malaysia.indeed.com/viewjob?jk=22e5181...</td>\n",
       "      <td>http://malaysia.indeed.com/job/security-guardp...</td>\n",
       "      <td>Security Guard/Pengawal Keselamatan</td>\n",
       "      <td>Chuan Sin Sdn Bhd</td>\n",
       "      <td>Taiping, M08, MY</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>2024-08-08</td>\n",
       "      <td>direct_data</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Guard</td>\n",
       "      <td>Ipoh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>77256fb4efac60cb</td>\n",
       "      <td>indeed</td>\n",
       "      <td>https://malaysia.indeed.com/viewjob?jk=77256fb...</td>\n",
       "      <td>http://malaysia.indeed.com/job/volunteer-teach...</td>\n",
       "      <td>VOLUNTEER TEACHER</td>\n",
       "      <td>Kelas Tuisyen Pro Elit</td>\n",
       "      <td>Ipoh, M08, MY</td>\n",
       "      <td>parttime, temporary</td>\n",
       "      <td>2024-08-09</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>Ipoh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>54c6d138a7fabf54</td>\n",
       "      <td>indeed</td>\n",
       "      <td>https://malaysia.indeed.com/viewjob?jk=54c6d13...</td>\n",
       "      <td>http://malaysia.indeed.com/job/procurement-exe...</td>\n",
       "      <td>PROCUREMENT EXECUTIVES</td>\n",
       "      <td>RECRUIT ADVANCE SDN BHD</td>\n",
       "      <td>Kuala Kangsar, M08, MY</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>2024-08-09</td>\n",
       "      <td>direct_data</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>software</td>\n",
       "      <td>Ipoh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>baef47abc000370a</td>\n",
       "      <td>indeed</td>\n",
       "      <td>https://malaysia.indeed.com/viewjob?jk=baef47a...</td>\n",
       "      <td>http://malaysia.indeed.com/job/graphic-designe...</td>\n",
       "      <td>Graphic Designer</td>\n",
       "      <td>IP Printing (M) Sdn Bhd</td>\n",
       "      <td>Ipoh, M08, MY</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>2024-08-09</td>\n",
       "      <td>direct_data</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>software</td>\n",
       "      <td>Ipoh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>fdd94d7a55aad3aa</td>\n",
       "      <td>indeed</td>\n",
       "      <td>https://malaysia.indeed.com/viewjob?jk=fdd94d7...</td>\n",
       "      <td>http://malaysia.indeed.com/job/procurement-exe...</td>\n",
       "      <td>PROCUREMENT EXECUTIVE @ EKSEKUTIF PEROLEHAN</td>\n",
       "      <td>RECRUIT ADVANCE SDN BHD</td>\n",
       "      <td>Kuala Kangsar, M08, MY</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>2024-08-08</td>\n",
       "      <td>direct_data</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>software</td>\n",
       "      <td>Ipoh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>b2e788974f398a7e</td>\n",
       "      <td>indeed</td>\n",
       "      <td>https://malaysia.indeed.com/viewjob?jk=b2e7889...</td>\n",
       "      <td>http://malaysia.indeed.com/job/sales-consultan...</td>\n",
       "      <td>Sales Consultant (Interior Design Studio)</td>\n",
       "      <td>Goldfields Concepts Sdn. Bhd.</td>\n",
       "      <td>Ipoh, M08, MY</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>2024-08-08</td>\n",
       "      <td>direct_data</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>software</td>\n",
       "      <td>Ipoh</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id    site  \\\n",
       "14   22e518104ecb405b  indeed   \n",
       "37   77256fb4efac60cb  indeed   \n",
       "142  54c6d138a7fabf54  indeed   \n",
       "143  baef47abc000370a  indeed   \n",
       "144  fdd94d7a55aad3aa  indeed   \n",
       "145  b2e788974f398a7e  indeed   \n",
       "\n",
       "                                               job_url  \\\n",
       "14   https://malaysia.indeed.com/viewjob?jk=22e5181...   \n",
       "37   https://malaysia.indeed.com/viewjob?jk=77256fb...   \n",
       "142  https://malaysia.indeed.com/viewjob?jk=54c6d13...   \n",
       "143  https://malaysia.indeed.com/viewjob?jk=baef47a...   \n",
       "144  https://malaysia.indeed.com/viewjob?jk=fdd94d7...   \n",
       "145  https://malaysia.indeed.com/viewjob?jk=b2e7889...   \n",
       "\n",
       "                                        job_url_direct  \\\n",
       "14   http://malaysia.indeed.com/job/security-guardp...   \n",
       "37   http://malaysia.indeed.com/job/volunteer-teach...   \n",
       "142  http://malaysia.indeed.com/job/procurement-exe...   \n",
       "143  http://malaysia.indeed.com/job/graphic-designe...   \n",
       "144  http://malaysia.indeed.com/job/procurement-exe...   \n",
       "145  http://malaysia.indeed.com/job/sales-consultan...   \n",
       "\n",
       "                                           title  \\\n",
       "14           Security Guard/Pengawal Keselamatan   \n",
       "37                             VOLUNTEER TEACHER   \n",
       "142                       PROCUREMENT EXECUTIVES   \n",
       "143                             Graphic Designer   \n",
       "144  PROCUREMENT EXECUTIVE @ EKSEKUTIF PEROLEHAN   \n",
       "145    Sales Consultant (Interior Design Studio)   \n",
       "\n",
       "                           company                location  \\\n",
       "14               Chuan Sin Sdn Bhd        Taiping, M08, MY   \n",
       "37          Kelas Tuisyen Pro Elit           Ipoh, M08, MY   \n",
       "142        RECRUIT ADVANCE SDN BHD  Kuala Kangsar, M08, MY   \n",
       "143        IP Printing (M) Sdn Bhd           Ipoh, M08, MY   \n",
       "144        RECRUIT ADVANCE SDN BHD  Kuala Kangsar, M08, MY   \n",
       "145  Goldfields Concepts Sdn. Bhd.           Ipoh, M08, MY   \n",
       "\n",
       "                job_type date_posted salary_source  ... company_addresses  \\\n",
       "14              fulltime  2024-08-08   direct_data  ...              None   \n",
       "37   parttime, temporary  2024-08-09          None  ...              None   \n",
       "142             fulltime  2024-08-09   direct_data  ...              None   \n",
       "143             fulltime  2024-08-09   direct_data  ...              None   \n",
       "144             fulltime  2024-08-08   direct_data  ...              None   \n",
       "145             fulltime  2024-08-08   direct_data  ...              None   \n",
       "\n",
       "     company_num_employees  company_revenue company_description  \\\n",
       "14                    None             None                None   \n",
       "37                    None             None                None   \n",
       "142                   None             None                None   \n",
       "143                   None             None                None   \n",
       "144                   None             None                None   \n",
       "145                   None             None                None   \n",
       "\n",
       "    logo_photo_url banner_photo_url ceo_name ceo_photo_url input_search_term  \\\n",
       "14            None             None     None          None             Guard   \n",
       "37            None             None     None          None           Teacher   \n",
       "142           None             None     None          None          software   \n",
       "143           None             None     None          None          software   \n",
       "144           None             None     None          None          software   \n",
       "145           None             None     None          None          software   \n",
       "\n",
       "    input_location  \n",
       "14            Ipoh  \n",
       "37            Ipoh  \n",
       "142           Ipoh  \n",
       "143           Ipoh  \n",
       "144           Ipoh  \n",
       "145           Ipoh  \n",
       "\n",
       "[6 rows x 33 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df[ results_df['input_location'] == 'Ipoh'  ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['product analyst', 'market analyst', 'business analyst']\""
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import streamlit as st\n",
    "from utils.text_extraction import extract_text_from_cv, extract_job_description_from_link, parse_file_contents\n",
    "from utils.cover_letter import generate_cover_letter, generate_feedback, create_search_terms\n",
    "from utils.key_validation import is_valid_key\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "import pandas as pd\n",
    "#import sqlite3\n",
    "from jobspy import scrape_jobs\n",
    "from st_aggrid import AgGrid, GridOptionsBuilder\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "\n",
    "#14/8 test\n",
    "search_term_list = create_search_terms('product analyst')\n",
    "\n",
    "search_term_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['product analyst', 'market analyst', 'business analyst']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import ast\n",
    "\n",
    "# Convert the string to a list\n",
    "search_term_list = ast.literal_eval(search_term_list)\n",
    "\n",
    "search_term_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}