scraping-for-latestnews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    developmentMode = True\n",
    "    callAPI = True\n",
    "    Local_API_Laravel = 'http://127.0.0.1:8000/api/news/store'\n",
    "    Live_API_Laravel = 'https://pakteki.com/api/news/store'\n",
    "    Live_API_Node = 'http://3.142.50.232:5000/api/news/all'\n",
    "    API = Live_API_Laravel\n",
    "    Dawn = True\n",
    "    Geo = True\n",
    "    Ary = True\n",
    "    Ptv = True\n",
    "    Express = True\n",
    "    Sama = True\n",
    "    Duniya = True\n",
    "    Bol = True\n",
    "    Abbtak = True\n",
    "    Ninetytwonews = True\n",
    "    Abbtak = True\n",
    "    Twentyfournews = False\n",
    "    Gnnnews = True\n",
    "    Dailypakistan = False\n",
    "    Newsone = True\n",
    "    Mashion = True\n",
    "    Mangobaaz = True\n",
    "    Sunday = True\n",
    "    Urdunews = True\n",
    "    Urdupoint = True\n",
    "    Tribune = True\n",
    "    Hellopakistanmag = True\n",
    "    Zaiqa = False\n",
    "    Islamicinfoenglish = False\n",
    "    Islamicinfourdu = False\n",
    "    Royal = True\n",
    "    Neonews = False\n",
    "    City42 = False\n",
    "    Jang = True\n",
    "\n",
    "    # libraries\n",
    "    from bs4 import BeautifulSoup,Comment\n",
    "    import requests\n",
    "    import csv\n",
    "    from datetime import date\n",
    "    import pandas as pd\n",
    "    import time\n",
    "\n",
    "    # Array stucture for storing data\n",
    "    columns = ['title','category', 'channel', 'permalink', 'image_src', 'date_time', 'description', 'fetching_date', 'language']\n",
    "    rows, cols = (0, 9)\n",
    "    data = [[0]*cols]*rows\n",
    "except:\n",
    "    print('exception in importing libraries ..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Dawn News...\n",
      "total news found on listting page : 30\n",
      "\n",
      "No:          1\n",
      "title:       Karachi's Covid-19 positivity rate tops 10pc as Omicron drives surge\n",
      "description: As the fifth wave of\n",
      "permalink:     https://www.dawn.com/news/1668143/karachis-covid-19-positivity-rate-tops-10pc-as-omicron-drives-surge\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d816eebb607.jpg\n",
      "category:    Covid\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       Election rallies cancelled as India sees Omicron surge\n",
      "description: NEW DELHI: Election \n",
      "permalink:     https://www.dawn.com/news/1668099/election-rallies-cancelled-as-india-sees-omicron-surge\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d7c3825fc61.jpg\n",
      "category:    Covid\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       Boris Johnson hits out at anti-vaxxers’ ‘mumbo jumbo’\n",
      "description: LONDON: British Prim\n",
      "permalink:     https://www.dawn.com/news/1668097/boris-johnson-hits-out-at-anti-vaxxers-mumbo-jumbo\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d7c4a1517b2.jpg\n",
      "category:    Covid\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       84-year-old Indian takes 11 Covid jabs\n",
      "description: NEW DELHI: An 84-yea\n",
      "permalink:     https://www.dawn.com/news/1668096/84-year-old-indian-takes-11-covid-jabs\n",
      "channel:     Dawn\n",
      "image:       \n",
      "category:    Covid\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       Omicron forces FM Qureshi to put off US visit\n",
      "description: WASHINGTON: Foreign \n",
      "permalink:     https://www.dawn.com/news/1668093/omicron-forces-fm-qureshi-to-put-off-us-visit\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d7bddaefc1d.jpg\n",
      "category:    Covid\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Novak Djokovic spends Orthodox Christmas in Australian detention amid legal fight\n",
      "description: World number one Nov\n",
      "permalink:     https://www.dawn.com/news/1668135/novak-djokovic-spends-orthodox-christmas-in-australian-detention-amid-legal-fight\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d7f939dbd8d.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       Ramiz Raja reiterates PCB needs to work on three key areas\n",
      "description: LAHORE: Pakistan Cri\n",
      "permalink:     https://www.dawn.com/news/1668039/ramiz-raja-reiterates-pcb-needs-to-work-on-three-key-areas\n",
      "channel:     Dawn\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       In touch with reality, GSV signs deal with Irish club for player development\n",
      "description: KARACHI: For all tal\n",
      "permalink:     https://www.dawn.com/news/1668037/in-touch-with-reality-gsv-signs-deal-with-irish-club-for-player-development\n",
      "channel:     Dawn\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Nadal wins opening match; Osaka also advances\n",
      "description: MELBOURNE: Rafael Na\n",
      "permalink:     https://www.dawn.com/news/1668033/nadal-wins-opening-match-osaka-also-advances\n",
      "channel:     Dawn\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       Women’s Ashes brought forward due to World Cup\n",
      "description: LONDON: The Women’s \n",
      "permalink:     https://www.dawn.com/news/1668038/womens-ashes-brought-forward-due-to-world-cup\n",
      "channel:     Dawn\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       South Africa skipper guides team to victory against India in second Test\n",
      "description: South Africa captain\n",
      "permalink:     https://www.dawn.com/news/1667989/south-africa-skipper-guides-team-to-victory-against-india-in-second-test\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d732cb7763e.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       Pakistan rejects 'tendentious' Indian claims about Saarc summit, occupied Kashmir\n",
      "description: The Foreign Office (\n",
      "permalink:     https://www.dawn.com/news/1668148/pakistan-rejects-tendentious-indian-claims-about-saarc-summit-occupied-kashmir\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d836d4c2751.png\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       Omicron forces FM Qureshi to put off US visit\n",
      "description: WASHINGTON: Foreign \n",
      "permalink:     https://www.dawn.com/news/1668093/omicron-forces-fm-qureshi-to-put-off-us-visit\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d7bddaefc1d.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       Dubai firm to build dry port in Indian-occupied Kashmir\n",
      "description: DUBAI: Dubai ports g\n",
      "permalink:     https://www.dawn.com/news/1668091/dubai-firm-to-build-dry-port-in-indian-occupied-kashmir\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d7c07dea9dd.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       Biden tears into Trump on anniversary of ‘insurrection’\n",
      "description:  WASHINGTON: Preside\n",
      "permalink:     https://www.dawn.com/news/1668098/biden-tears-into-trump-on-anniversary-of-insurrection\n",
      "channel:     Dawn\n",
      "image:       \n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       Bajwa underscores importance of peace in Afghanistan\n",
      "description: RAWALPINDI: Chief of\n",
      "permalink:     https://www.dawn.com/news/1668071/bajwa-underscores-importance-of-peace-in-afghanistan\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d7b6ec2b92c.png\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       Election rallies cancelled as India sees Omicron surge\n",
      "description: NEW DELHI: Election \n",
      "permalink:     https://www.dawn.com/news/1668099/election-rallies-cancelled-as-india-sees-omicron-surge\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d7c3825fc61.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       Future of e-commerce strong, Alibaba country head tells Pakistanis\n",
      "description: LAHORE: As part of i\n",
      "permalink:     https://www.dawn.com/news/1668080/future-of-e-commerce-strong-alibaba-country-head-tells-pakistanis\n",
      "channel:     Dawn\n",
      "image:       \n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       Nasa deploys Webb telescope’s sunshield\n",
      "description: CAPE CANAVERAL: The \n",
      "permalink:     https://www.dawn.com/news/1667681/nasa-deploys-webb-telescopes-sunshield\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d5252b124ca.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       Govt mulls revisiting IT sector tax measures\n",
      "description: ISLAMABAD: Amid a  s\n",
      "permalink:     https://www.dawn.com/news/1667695/govt-mulls-revisiting-it-sector-tax-measures\n",
      "channel:     Dawn\n",
      "image:       \n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       Apple becomes first company to hit $3 trillion market value, then slips\n",
      "description: Apple Inc on Monday \n",
      "permalink:     https://www.dawn.com/news/1667594/apple-becomes-first-company-to-hit-3-trillion-market-value-then-slips\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2022/01/61d3efc17edd4.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       An unstoppable tech frenzy\n",
      "description:  If and when the his\n",
      "permalink:     https://www.dawn.com/news/1667395/the-startup-capital-frenzy-is-not-stopping-any-time-soon\n",
      "channel:     Dawn\n",
      "image:       \n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       Palestinians raise alarm over Facebook content 'suppression'\n",
      "description: Palestinian journali\n",
      "permalink:     https://www.dawn.com/news/1666820/palestinians-raise-alarm-over-facebook-content-suppression\n",
      "channel:     Dawn\n",
      "image:       https://i.dawn.com/primary/2021/12/61cead1de2806.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 23\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources = [\n",
    "                [\"https://www.dawn.com/trends/coronavirus\", \"Covid\", \"Dawn\"],\n",
    "               [\"https://www.dawn.com/sport\", \"Sports\", \"Dawn\"], \n",
    "               [\"https://www.dawn.com/world\", \"International\", \"Dawn\"],\n",
    "               [\"https://images.dawn.com/art-culture\", \"LifeStyle-Culture\", \"Dawn\"],\n",
    "               [\"https://www.dawn.com/tech\", \"IT-Science\", \"Dawn\"]\n",
    "               ]\n",
    "    if Dawn:\n",
    "        counter = 0\n",
    "        print('Getting Dawn News...')\n",
    "        i = 1\n",
    "        permalinkDawnNews = []\n",
    "        for source in sources:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('article', class_='box'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = article.find('a', class_='story__link')['href']\n",
    "                        permalinkDawnNews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkDawnNews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "\n",
    "            for div in soup.find_all('div',class_='w-full max-w-screen-md xl:pr-4'):\n",
    "                try:\n",
    "                    detail_title = div.find('div',class_='template__header').h2.a.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.find('div',class_='template__header').find('div',class_='slideshow').find('div',class_='slideshow__slide').figure.div.picture.img['src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='template__main').find('div',class_='story__content').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink':detail_page_source[0] ,\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:20])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter)) \n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting ARY News...\n",
      "total news found on listting page : 30\n",
      "\n",
      "No:          1\n",
      "title:       Dubai Economic Zones Authority announces new leadership team\n",
      "description: Sheikh Ahmed bin Saeed Al Maktoum, Chairman of Dubai Integrated Economic Zones A\n",
      "permalink:   https://arynews.tv/diez-announces-new-leadership-team/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/DIEZ.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       India’s COVID-19 cases set for new highs as Omicron spreads\n",
      "description: NEW DELHI: India’s daily COVID-19 cases jumped to 117,100 on Friday, a five-fold\n",
      "permalink:   https://arynews.tv/india-covid-19-cases-omicron-variant-spread/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/04/2-bed-India-coronavirus.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       CSTO peacekeeping troops reach Kazakhstan\n",
      "description: ALMATY: The Collective Security Treaty Organization (CSTO) has sent peacekeeping\n",
      "permalink:   https://arynews.tv/csto-peacekeeping-troops-reach-kazakhstan/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/kazakhstan-peacekeeping-troops-csto.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       Biden assails Trump in speech marking anniversary of Capitol riot\n",
      "description: President Joe Biden tore into Donald Trump Thursday on the first anniversary of \n",
      "permalink:   https://arynews.tv/biden-assails-trump-anniversary-capitol-riot/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/12/Biden.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       Calling Omicron ‘mild’ a mistake, warns WHO\n",
      "description: The Omicron variant of Covid-19 is killing people across the globe and should no\n",
      "permalink:   https://arynews.tv/calling-omicron-mild-mistake-warns-who/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2020/10/WHO.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Novak Djokovic denied entry to Australia over vaccination status\n",
      "description: MELBOURNE: World number one tennis player Novak Djokovic was denied entry into A\n",
      "permalink:   https://arynews.tv/novak-djokovic-denied-entry-australia-vaccination-status/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/Novak-Djokovic.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       #CES2022: Metaverse gets touch of reality\n",
      "description: A jacket equipped with sensors that let wearers feel hugs or even punches in vir\n",
      "permalink:   https://arynews.tv/ces-2022-metaverse-gets-touch-of-reality/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/metaverse.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       Sinopharm protein-based booster stronger against Omicron\n",
      "description: A protein-based COVID-19 vaccine made by Sinopharm, when given as a booster afte\n",
      "permalink:   https://arynews.tv/sinopharm-protein-based-booster-stronger-omicron/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/06/Sinopharm.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Change your car’s colour with an app!\n",
      "description: German carmaker BMW has unveiled the world’s first “colour-changing” car at the \n",
      "permalink:   https://arynews.tv/change-your-cars-colour-with-an-app/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/car.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       Here is the guide to install PUBG Mobile Lite 0.22.1 APK update\n",
      "description: Tencent’s latest PUBG Mobile Lite update has recently hit the servers and brough\n",
      "permalink:   https://arynews.tv/pubg-mobile-lite-0-22-1-apk-update/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/pubg.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       Google Maps helps police capture mafia fugitive\n",
      "description: ROME: Italian police has caught a top mafia fugitive who had been on the run for\n",
      "permalink:   https://arynews.tv/google-maps-helps-italian-police-capture-mafia-fugitive/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/google-maps-1.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       WATCH: Hyundai says ‘metamobility’ will link real and virtual worlds\n",
      "description: Hyundai Motor envisions an interactive and partly virtual future it calls “metam\n",
      "permalink:   https://arynews.tv/hyundai-metamobility-virtual-worlds-video/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/Hyundai-metamobility.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       Covid vaccination mandatory for students in Sindh\n",
      "description: KARACHI: With the fifth wave of the coronavirus intensifying in the country, the\n",
      "permalink:   https://arynews.tv/covid-vaccination-mandatory-for-students-in-sindh/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/01/kpk-schools.jpg\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       India’s COVID-19 cases set for new highs as Omicron spreads\n",
      "description: NEW DELHI: India’s daily COVID-19 cases jumped to 117,100 on Friday, a five-fold\n",
      "permalink:   https://arynews.tv/india-covid-19-cases-omicron-variant-spread/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/04/2-bed-India-coronavirus.jpg\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       Omicron: Has govt announced restrictions for Sindh?\n",
      "description: KARACHI: With the provincial authorities mulling over new restrictions to curb r\n",
      "permalink:   https://arynews.tv/omicron-has-govt-announced-restrictions-for-sindh/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/270350630_1585291415167461_7151279060193476055_n.png\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       Sinopharm protein-based booster stronger against Omicron\n",
      "description: A protein-based COVID-19 vaccine made by Sinopharm, when given as a booster afte\n",
      "permalink:   https://arynews.tv/sinopharm-protein-based-booster-stronger-omicron/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/06/Sinopharm.jpg\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       136 more found infected with Omicron in Karachi\n",
      "description: KARACHI: As many as 136 more people were found infected with the Omicron variant\n",
      "permalink:   https://arynews.tv/136-more-found-infected-with-omicron-in-karachi/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/12/omicron-covid-variant-vs-delta-1.jpg\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       COVID-19: Sindh reduces timings for vaccination centres\n",
      "description: KARACHI: Timings for the Mega COVID-19 vaccination centres has been reduced to 1\n",
      "permalink:   https://arynews.tv/covid-19-sindh-reduces-timings-for-vaccination-centres/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/10/vaccine.jpg\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       Dubai Economic Zones Authority announces new leadership team\n",
      "description: Sheikh Ahmed bin Saeed Al Maktoum, Chairman of Dubai Integrated Economic Zones A\n",
      "permalink:   https://arynews.tv/diez-announces-new-leadership-team/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/DIEZ.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       #CES2022: Metaverse gets touch of reality\n",
      "description: A jacket equipped with sensors that let wearers feel hugs or even punches in vir\n",
      "permalink:   https://arynews.tv/ces-2022-metaverse-gets-touch-of-reality/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/metaverse.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       Rupee recoups losses against US dollar\n",
      "description: KARACHI: The Pakistani rupee recouped some losses against the United States (US)\n",
      "permalink:   https://arynews.tv/rupee-recoups-losses-against-us-dollar/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/11/US-dollar-pak-rupee.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       Silk Route Reconnect raises Pakistan’s exports to Central Asia: Dawood\n",
      "description: ISLAMABAD: Adviser to Prime Minister on Commerce Abdul Razak Dawood on Friday an\n",
      "permalink:   https://arynews.tv/silk-route-reconnect-raises-pakistans-exports-to-central-asia-dawood/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/07/Afghanistan-Pakistan-Transit-Trade-Agreement-1.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       FBR collected Rs4,745 billion tax in 2020-21: Finance ministry\n",
      "description: ISLAMABAD: The Federal Board of Revenue (FBR) collected tax of Rs4,745 billion d\n",
      "permalink:   https://arynews.tv/fbr-collected-rs4745-billion-tax-in-2020-21-finance-ministry/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2020/06/FBR-Pakistan-1-750x369-1.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       ‘Stock market claims highest profit of 10 years in 2021’\n",
      "description: ISLAMABAD: Finance Minister Shaukat Tarin on Friday shared that Pakistan’s stock\n",
      "permalink:   https://arynews.tv/stock-market-claims-highest-profit-of-10-years-in-2021/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2020/09/Pakistan-Stock-Exchange.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          25\n",
      "title:       Police recover 20 hand grenades from Karachi’s Garden\n",
      "description: KARACHI: Police on Friday claimed to have foiled a terror bid and recovered 20 h\n",
      "permalink:   https://arynews.tv/police-recover-hand-grenades-from-karachi-garden/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/grenade.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          26\n",
      "title:       Pakistan rejects India’s ‘false’ claims about SAARC, IIOJK\n",
      "description: ISLAMABAD: Pakistan on Friday categorically rejected India’s “false claims and t\n",
      "permalink:   https://arynews.tv/pakistan-rejects-indias-false-claims-about-saarc-iiojk/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/10/foreign-office-1-1.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          27\n",
      "title:       Covid vaccination mandatory for students in Sindh\n",
      "description: KARACHI: With the fifth wave of the coronavirus intensifying in the country, the\n",
      "permalink:   https://arynews.tv/covid-vaccination-mandatory-for-students-in-sindh/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/01/kpk-schools.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          28\n",
      "title:       Omicron: Has govt announced restrictions for Sindh?\n",
      "description: KARACHI: With the provincial authorities mulling over new restrictions to curb r\n",
      "permalink:   https://arynews.tv/omicron-has-govt-announced-restrictions-for-sindh/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2022/01/270350630_1585291415167461_7151279060193476055_n.png\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          29\n",
      "title:       Four muggers arrested in two Islamabad police encounters in a day\n",
      "description: ISLAMABAD: Islamabad police on Friday claimed to have apprehended four muggers w\n",
      "permalink:   https://arynews.tv/muggers-islamabad-police-encounters-robbery/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2017/09/Islamabad-police.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          30\n",
      "title:       136 more found infected with Omicron in Karachi\n",
      "description: KARACHI: As many as 136 more people were found infected with the Omicron variant\n",
      "permalink:   https://arynews.tv/136-more-found-infected-with-omicron-in-karachi/\n",
      "channel:     ARY\n",
      "image:       https://arynews.tv/wp-content/uploads/2021/12/omicron-covid-variant-vs-delta-1.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 30\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}\n",
    "    sources2 = [\n",
    "                [\"https://arynews.tv/category/international-2\", \"International\", \"ARY\"],\n",
    "                [\"https://arynews.tv/category/sci-techno/\", \"IT-Science\", \"ARY\"], \n",
    "                [\"https://arynews.tv/category/health-2/\", \"Health\", \"ARY\"],\n",
    "                [\"https://arynews.tv/category/business/\",\"Business\",\"ARY\"],\n",
    "                [\"https://arynews.tv/category/pakistan/\",\"National-local\",\"ARY\"],\n",
    "               ]\n",
    "    if Ary == True:\n",
    "        print('Getting ARY News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayARY = []\n",
    "        for source in sources2:\n",
    "            sourceHTML = requests.get(source[0],headers=headers).text\n",
    "            soup = BeautifulSoup(sourceHTML,'lxml')\n",
    "            for div in soup.find_all('div', class_='td-module-container'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = div.find('div',class_='td-module-meta-info').h3.a['href']\n",
    "                        permalinkArrayARY.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayARY\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0],headers=headers).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div', class_='tdi_74'):\n",
    "                try:\n",
    "                    detail_title = div.find('div',class_='tdb_title').div.h2.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:  \n",
    "                    start = div.find('div',class_='tdb_single_bg_featured_image').style.find_next_sibling().text.find('url')\n",
    "                    end = div.find('div',class_='tdb_single_bg_featured_image').style.find_next_sibling().text.find(')')\n",
    "                    high_resolution_img_data = div.find('div',class_='tdb_single_bg_featured_image').style.find_next_sibling().text\n",
    "                    high_resolution_image = high_resolution_img_data[start:end][5:-1]\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.div.div.find('div',class_='td_block_wrap tdb_single_content tdi_78 td-pb-border-top td_block_template_1 td-post-content test tagdiv-type').div.find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting GEO News...\n",
      "total news found on listting page : 30\n",
      "\n",
      "No:          1\n",
      "title:       Dog custody: Spain to consider pets' welfare in divorce battles\n",
      "description: MADRID: Spain will consider a pet's welfare when couples divorce or break up fro\n",
      "permalink:     https://www.geo.tv/latest/392113-dog-custody-spain-to-consider-pets-welfare-in-divorce-battles\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392113_1595108_updates.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       Kazakh president says constitutional order has mostly been restored\n",
      "description: ALMATY: Kazakh President Kassym-Jomart Tokayev said on Friday that constitutiona\n",
      "permalink:     https://www.geo.tv/latest/392058-kazakh-president-says-constitutional-order-has-mostly-been-restored\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392058_8920117_updates.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       Omicron may be less severe in young and old, but not 'mild', WHO says\n",
      "description: The more infectious Omicron variant of COVID-19 appears to produce less severe d\n",
      "permalink:     https://www.geo.tv/latest/392038-omicron-may-be-less-severe-in-young-and-old-but-not-mild-who-says\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392038_4801539_updates.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       Russia sends troops to put down Kazakhstan uprising as fresh violence erupts\n",
      "description: ALMATY: Fresh violence erupted in Kazakhstan's main city of Almaty on Thursday a\n",
      "permalink:     https://www.geo.tv/latest/392029-russia-sends-troops-to-put-down-kazakhstan-uprising-as-fresh-violence\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392029_9894714_updates.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       From stability to turmoil — what's going on in Kazakhstan\n",
      "description: Soldiers shot at protesters in Kazakhstan's biggest city Almaty on Thursday afte\n",
      "permalink:     https://www.geo.tv/latest/391971-from-stability-to-turmoil-whats-going-on-in-kazakhstan\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-06/391971_9015243_updates.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       One year on, Trump backers recall 'incredible' Capitol siege\n",
      "description: One year ago to the day, they descended upon Washington in the thousands, gather\n",
      "permalink:     https://www.geo.tv/latest/391972-one-year-on-trump-backers-recall-incredible-capitol-siege\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-06/391972_9618359_updates.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       Apple becomes first company to hit $3 trillion market value, then slips\n",
      "description: Apple Inc on Monday became the first company to hit a $3 trillion stock market v\n",
      "permalink:     https://www.geo.tv/latest/391542-apple-becomes-first-company-to-hit-3-trillion-market-value-then-slips\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-04/391542_6393253_updates.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       Apple becomes 1st US company to reach $3 tn valuation\n",
      "description: NEW YORK: Apple became the first US company to hit $3 trillion in market value, \n",
      "permalink:     https://www.geo.tv/latest/391433-apple-becomes-1st-us-company-to-reach-3-tn-valuation\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-04/391433_3065094_updates.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Denmark aims for all domestic flights to be green by 2030\n",
      "description: COPENHAGEN: Denmark's government has set an ambitious target of making all Danis\n",
      "permalink:     https://www.geo.tv/latest/391082-denmark-aims-for-all-domestic-flights-to-be-green-by-2030\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-02/391082_5572808_updates.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       CES tech show shutting down a day early amid Covid surge\n",
      "description: LAS VEGAS: The CES tech show announced Friday that the Las Vegas event next week\n",
      "permalink:     https://www.geo.tv/latest/390904-ces-tech-show-shutting-down-a-day-early-amid-covid-surge\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-01/390904_5598324_updates.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       2021 — The wildest year yet for cryptocurrencies\n",
      "description: LONDON: Bitcoin close to $70,000, \"memecoins\" worth billions of dollars, a block\n",
      "permalink:     https://www.geo.tv/latest/390873-2021-cryptocurrencies-wildest-year-yet\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2021-12-31/390873_6451172_updates.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       TikTok breaks silence after two TikTokers arrested for 'accidentally shooting man'\n",
      "description:  Famous video-sharing platform TikTok on Friday broke its silence after police a\n",
      "permalink:     https://www.geo.tv/latest/390884-tiktok-breaks-silence-after-two-tiktokers-arrested-for-accidentally-shooting-man\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2021-12-31/390884_318090_updates.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       WATCH: Ben Stokes survives as bails stay intact after ball hits stumps\n",
      "description: Despite getting hit on stumps during the fourth Test match of the Ashes series a\n",
      "permalink:     https://www.geo.tv/latest/392067-the-ashes-ben-stokes-survives-as-bails-stay-intact\n",
      "channel:     GEO\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       PSL 7: How do Inzamam, Wahab see Shaheen's appointment as Qalandars' skipper?\n",
      "description: LAHORE: Ex-Pakistan skipper Inzamam-ul-Haq and left-arm fast bowler Wahab Riaz h\n",
      "permalink:     https://www.geo.tv/latest/392047-psl-7-how-do-inzamam-wahab-see-shaheens-appointment-as-qalandars-skipper\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392047_8336595_updates.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       Imran Khan retired from Test cricket on this day: See his top records here\n",
      "description: The former Pakistan captain, Imran Khan, retired from his 21-year Test cricket c\n",
      "permalink:     https://www.geo.tv/latest/392043-imran-khan-retired-from-test-cricket-on-this-day-see-top-records\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392043_1356664_updates.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       Abid Ali to travel to Lahore today following successful rehab\n",
      "description: KARACHI: Pakistan Test cricketer will finally depart for his hometown, Lahore, t\n",
      "permalink:     https://www.geo.tv/latest/392039-abid-ali-to-travel-to-lahore-today-after-successful-rehab\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392039_1126589_updates.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       PTV fails to submit details of ARY deal to LHC in PSL rights case\n",
      "description: LAHORE: State broadcaster Pakistan Television (PTV) has failed to submit details\n",
      "permalink:     https://www.geo.tv/latest/392033-ptv-fails-to-submit-details-of-ary-deal-to-lhc-in-psl-rights-case\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392033_7738742_updates.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       Djokovic wins deportation delay after Australia cancels visa\n",
      "description: MELBOURNE: Tennis world number one Novak Djokovic won a temporary reprieve in hi\n",
      "permalink:     https://www.geo.tv/latest/392001-djokovic-wins-deportation-delay-after-australia-cancels-visa\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-06/392001_9522645_updates.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       Sandra Bullock thinks 'a lot of people wouldn't be working' if it weren't for Netflix\n",
      "description: Sandra Bullock expressed her views on streaming services as she complimented Net\n",
      "permalink:     https://www.geo.tv/latest/392123-sandra-bullock-thinks-a-lot-of-people-wouldnt-be-working-if-it-werent-for-netflix\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392123_3068427_updates.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       Queen is pro at technology, has secret Facebook account: Report\n",
      "description: Queen Elizabeth II is good at adapting technology with the possibility of having\n",
      "permalink:     https://www.geo.tv/latest/392117-queen-is-pro-at-technology-has-private-facebook-account-report\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392117_4814859_updates.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       Adele feels ‘ready for 2022’, drops teaser of ‘Oh My God’ music video: Watch\n",
      "description: Adele has 'rested' well and now she is gearing up to re-set all records with her\n",
      "permalink:     https://www.geo.tv/latest/392114-adele-feels-ready-for-2022-drops-teaser-of-oh-my-god-music-video-watch\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392114_5603886_updates.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       Royal photographer reveals FIRST-EVER photo he took of Kate Middleton in 2005\n",
      "description: A royal photographer has revealed the first ever photograph he took of Kate Midd\n",
      "permalink:     https://www.geo.tv/latest/392112-royal-photographer-reveals-first-ever-photo-he-took-of-kate-middleton-in-2005\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392112_3781947_updates.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       Tom Holland recalls pitching for James Bond origin film: ‘dream of a young kid’\n",
      "description: Tom Holland recently recalled his dream of starring in James Bond movie as he re\n",
      "permalink:     https://www.geo.tv/latest/392106-tom-holland-recalls-pitching-for-james-bond-origin-film-dream-of-a-young-kid\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392106_5082875_updates.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       Andrew Garfield says he is ‘definitely open’ to return as ‘Spider-Man’\n",
      "description: Andrew Garfield has finally sounded off on his surprise appearance in the blockb\n",
      "permalink:     https://www.geo.tv/latest/392092-andrew-garfield-is-definitely-open-to-return-as-spider-man\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392092_9755387_updates.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          25\n",
      "title:       Sales tax on goods falls under federal govt's jurisdiction: Tarin\n",
      "description: ISLAMABAD: Federal Minister for Finance and Revenue Shaukat Tarin on Thursday cl\n",
      "permalink:     https://www.geo.tv/latest/392119-sales-tax-on-goods-falls-under-federal-govts-jurisdiction-tarin\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392119_2546683_updates.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          26\n",
      "title:       Rupee rebounds as SBP curbs certain outflows\n",
      "description: KARACHI: Pakistan currency recovered slightly against the US dollar on the last \n",
      "permalink:     https://www.geo.tv/latest/392095-rupee-rebounds-as-sbp-curbs-certain-outflows\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-07/392095_2941815_updates.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          27\n",
      "title:       Senate body rejects several finance bill proposals\n",
      "description: ISLAMABAD: The Federal Board of Revenue (FBR) on Thursday sought the right from \n",
      "permalink:     https://www.geo.tv/latest/391905-senate-body-rejects-several-finance-bill-proposals\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-06/391905_2645358_updates.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          28\n",
      "title:       Gold cuts losses as rupee slips against US dollar\n",
      "description: KARACHI: Gold prices in Pakistan were confined to a small range in seasonally-qu\n",
      "permalink:     https://www.geo.tv/latest/391931-gold-cuts-losses-as-rupee-slips-against-us-dollar\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-06/391931_4935979_updates.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          29\n",
      "title:       Rupee weakens by 0.10% against US dollar\n",
      "description: KARACHI: After a day's respite, the Pakistani rupee continued to lose its ground\n",
      "permalink:     https://www.geo.tv/latest/391897-rupee-weakens-by-010-against-us-dollar\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-06/391897_3985005_updates.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          30\n",
      "title:       'Mini-budget' an attempt to bring tsunami of inflation: Senate committee\n",
      "description:  ISLAMABAD: The Senate Standing Committee on Finance has termed the \"mini-budget\n",
      "permalink:     https://www.geo.tv/latest/391814-mini-budget-an-attempt-to-bring-tsunami-of-inflation-senate-standing-committee\n",
      "channel:     GEO\n",
      "image:       https://www.geo.tv/assets/uploads/updates/2022-01-06/391814_8836195_updates.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 30\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources3 = [\n",
    "                [\"https://www.geo.tv/category/world\", \"International\", \"GEO\"],\n",
    "                [\"https://www.geo.tv/category/sci-tech\", \"IT-Science\", \"GEO\"], \n",
    "                [\"https://www.geo.tv/category/sports\", \"Sports\", \"GEO\"],\n",
    "                [\"https://www.geo.tv/category/entertainment\", \"Entertainment\", \"GEO\"],\n",
    "                [\"https://www.geo.tv/category/business\",\"Business\",\"GEO\"],\n",
    "                [\"https://www.geo.tv/food\",\"Gourmet\",\"GEO\"],\n",
    "               ]\n",
    "    if Geo == True:\n",
    "        print('Getting GEO News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayGeoNews = []\n",
    "        for source in sources3:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div', class_='col-xs-6 col-sm-6 col-lg-6 col-md-6 singleBlock'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = div.div.ul.li.a['href']\n",
    "                        permalinkArrayGeoNews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayGeoNews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='column-right'):\n",
    "                try:\n",
    "                    detail_title = div.find('div',class_='left').find('div',class_='story-area').h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.find('div',class_='story-area').find('div',class_='content-area').find('div',class_='medium-insert-images').figure.img['src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='left').find('div',class_='story-area').find('div',class_='content-area').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:     ' + detail_page_source[0].strip())\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image.strip())\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting PTV News...\n",
      "total news found on listting page : 18\n",
      "\n",
      "No:          1\n",
      "title:       Govt Importing Wheat, Sugar To Ensure Smooth Supply At Reasonable Prices, Senate Told\n",
      "description:  The Senate was informed on Friday that a pilot project for 7th Population and H\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37407\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-07-2022-15.12.36_senate.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       Rain-Affected Areas Of Balochistan:\n",
      "Pakistan Navy Continues Relief Operations In Affected Areas\n",
      "description:  Pakistan Navy continues relief operations in remote coastal areas of Balochista\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37405\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-07-2022-13.17.13_pak navy flood fff.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       Govt To Hold LG Elections In Punjab In Transparent Manner: Chaudhry Fawad\n",
      "description:  Talking to media In Islamabad, the Information Minister said the government is \n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37403\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-07-2022-13.09.31_fawad chaudhry meeting 4544.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       Pakistan Stock Market Claims Highest Profit In 10 Years: Chaudhry Fawad\n",
      "description:  Minister For Information And Broadcasting Chaudhry Fawad Hussain has said that \n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37396\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-07-2022-12.10.28_fawad 27 new 22222.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       Senate Session:\n",
      "Pilot Project For 7th Population To Begin From May 15\n",
      "description:  The Senate was informed that a Pilot Project for 7th Population And Housing Cen\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37395\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-07-2022-12.07.42_senate 27 today.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Pakistan Wants To Expand Bilateral Cooperation With Greece In Diverse Fields: President\n",
      "description:  President Dr. Arif Alvi says Pakistan highly values its ties with Greece and wa\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37393\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-06-2022-15.43.24_presegrece.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       2nd Test In Johannesburg:\n",
      "South Africa Beat India By 7 Wickets To Level Series 1-1\n",
      "description:  South Africa defeated India by seven wickets in the second cricket test in Joha\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37401\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-07-2022-12.56.41_ckt sa vz india.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       4th Ashes Test:\n",
      "England Woes Continue In First Innings\n",
      "description:  England's disastrous Ashes in Australia continues as the tourists are strugglin\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37400\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-07-2022-12.53.37_cktb ashes eng aus.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Johannesburg Test\n",
      "South Africa Launch Tough Chase Against India\n",
      "description:  South African Captain Dean Elgar defied a hostile Indian Bowling attack on a di\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37391\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-06-2022-12.20.33_johansberg.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       ICC Test Rankings : Babar Azam Beats Kohli To Gain 8th Position\n",
      "description:  ICC Test Rankings : Babar Azam Beats Kohli To Gain 8th PositionPakistan Cricket\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37381\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-05-2022-20.26.25_BABAR AZAM..jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       Bangladesh Make History : Ban Beat New Zealand For First Time In Test Cricket\n",
      "description:  Bangladesh beat New Zealand for the first time in test cricket with a sensation\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37380\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-05-2022-20.18.19_222.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       Tauranga Test:\n",
      "Bangladesh Beat New Zealand For First Time In Test Cricket\n",
      "description:  Bangladesh beat New Zealand for the first time in test cricket with a sensation\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37364\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-05-2022-12.59.08_ban vs newzealand win 5.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       Kazakhstan Unrest:\n",
      "President Says Order Mostly Restored\n",
      "description:  Kazakhstan's President said order has been mostly restored in the country after\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37399\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-07-2022-12.50.52_kazikh protests.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       US, Japan Move Closer On Defence Amid Hypersonic Threat\n",
      "description:  The United States and Japan are set to sign a new Defence Collaboration Deal to\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37398\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-07-2022-12.47.44_japan us virtual.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       Missile Tests\n",
      "North Korea Successfully Tested Hypersonic Missile\n",
      "description:  North Korea successfully tested a hypersonic missile, its first major weapons t\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37386\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-06-2022-11.00.47_northlorea.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       Coronavirus Pandemic : Swedish King And Queen Test Positive For Covid-19\n",
      "description:  Sweden's King Carl XVI Gustaf and Queen Silvia have tested positive for covid-1\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37382\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-05-2022-21.56.11_22.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       North Korea Launches 'Unidentified Projectile' Into Sea\n",
      "description:  North Korea has fired what has been described as an Unidentified Projectile int\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37365\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-05-2022-13.09.03_north korea.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       U.S. Covid Concerns : Country Sets Global Record Of Over 1m Daily Cases\n",
      "description:  The United States recorded more than one million covid-19 cases, as the Omicron\n",
      "permalink:   https://ptv.com.pk/ptvWorld/engNewsDetail/37356\n",
      "channel:     PTV\n",
      "image:       https://ptv.com.pk/newsimages/Jan-04-2022-20.56.02_WORLD (47).jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 18\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources4 = [\n",
    "                [\"https://ptv.com.pk/ptvWorld/news/National\", \"National-Local\", \"PTV\"],\n",
    "                [\"https://ptv.com.pk/ptvWorld/news/Cricket\", \"Sports\", \"PTV\"], \n",
    "                [\"https://ptv.com.pk/ptvWorld/news/International\", \"International\", \"PTV\"]\n",
    "                ]\n",
    "    if Ptv == True:\n",
    "        print('Getting PTV News...')\n",
    "        counter = 0\n",
    "        i = 1\n",
    "        permalinkArrayPTV = []\n",
    "        for source in sources4:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('ul', class_='big'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = article.li.div.div.h4.a['href']\n",
    "                        permalinkArrayPTV.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayPTV\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='page'):\n",
    "                try:\n",
    "                    detail_title = div.find('div',class_='row').find('div',class_='column_2_3').find('div',class_='row').div.div.h3.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.find('div',class_='row').find('div',class_='column_2_3').find('div',class_='row').div.img['src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description = div.find('div',class_='row').find('div',class_='column_2_3').find('div',class_='row').div.div.div.p.text.replace('\\n','')\n",
    "                except:\n",
    "                    detail_description = '.'\n",
    "                counter = counter + 1\n",
    "                language = 'English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': permalink,\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Express News...\n",
      "total news found on listting page : 0\n",
      "\n",
      "No:          1\n",
      "title:       پہلی مرتبہ دماغ میں اوسی ڈی کی وجہ بننے والا برقی عمل دریافت\n",
      "description: سائنسدانوں نے پہلی مرتبہ دماغی برقی سرگرمی معلوم کی ہے جو او سی ڈی کی وجہ بنتی ہ\n",
      "permalink:   https://www.express.pk/story/2267755/9812\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2267755-brainmain-1641414569-254-640x480.jpg\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       ڈیمنشیا روکنے والے اسپرے کی انسانی آزمائش جلد متوقع\n",
      "description: جاپانی سائنسدانوں نے ایک اینٹی بایوٹک دوا اور اینٹی آکسیڈنٹ پر مبنی ناک کا اسپر\n",
      "permalink:   https://www.express.pk/story/2267295/9812\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2267295-nasal-1641320354-750-640x480.gif\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       شکرقندی کولیسٹرول گھٹانے میں انتہائی مفید\n",
      "description: شکرقندی کولیسٹرول گھٹانے والی نہایت تیربہدف غذا ہے۔ فوٹو: فائل واشنگٹن:ایک بہت س\n",
      "permalink:   https://www.express.pk/story/2266964/9812\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2266964-sweets-1641240167-408-640x480.gif\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       فرانس میں کورونا وائرس کا نیا ویریئنٹ سامنے آگیا\n",
      "description: فرانس میں اب تک کورونا کی اس نئی شکل کے 12 کیسز سامنے آئے ہیں، فوٹو: فائل پیرس:\n",
      "permalink:   https://www.express.pk/story/2267566/9812\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2267566-newcovidvarientdedectedinfrance-1641388996-131-640x480.jpg\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       ملیٹھی ایک جادوئی شفائی جڑی بوٹی بھی ہے\n",
      "description: ملیٹھی کو سپرفوڈ کہا جاتا ہے جو کئی بیماریوں کا شافی علاج بھی ہے۔ فوٹو: فائل کرا\n",
      "permalink:   https://www.express.pk/story/2266928/9812\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2266928-mulethi-1641232805-511-640x480.gif\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       ویاگرا نے کووِڈ 19 سے شدید متاثرہ خاتون کی جان بچا لی\n",
      "description: وہ پہلی مریضہ ہیں جن پر کووِڈ 19 کے علاج کےلیے ویاگرا آزمائی گئی ہے۔ (فوٹو: انٹ\n",
      "permalink:   https://www.express.pk/story/2267068/9812\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2267068-covidviagrax-1641281809-914-640x480.jpg\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       قذافی اسٹیڈیم کے باہر پی ایس ایل ٹرافی کی نقل سجا دی گئی\n",
      "description: پی ایس ایل 7 کا فائنل لاہورمیں ہوگا:فوٹو:فائل لاہور:پاکستان سپرلیگ کے 7 ویں ایڈی\n",
      "permalink:   https://www.express.pk/story/2268324/16\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268324-psltrophy-1641546520-461-640x480.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       اعصام الحق میلبورن ٹورنامنٹ کے سیمی فائنل میں پہنچ گئے\n",
      "description: میلبورن:پاکستانی ٹینس اسٹار اعصام الحق کی اے ٹی پی میلبورن ٹورنامنٹ میں فتوحات ک\n",
      "permalink:   https://www.express.pk/story/2268311/16\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/gettyimages-1232068383-1-1641543511-468-640x480.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       محمد رضوان 2021 کے سب سے قیمتی کھلاڑی قرار\n",
      "description: محمد رضوان نے ٹی ٹونٹی انٹرنیشنل میں ریکارڈ ساز 1326 رنز اسکور کیے (فائل فوٹو) ل\n",
      "permalink:   https://www.express.pk/story/2268051/16\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268051-muhammadrizwanxx-1641481300-638-640x480.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       پی سی بی کا کھلاڑیوں کو ویکسین کی بوسٹر ڈوز لگوانے کا فیصلہ\n",
      "description: کھلاڑیوں کو فائزر ویکسین کے بوسٹر ڈوز لگائے جائیں گے، کرکٹ بورڈ - فوٹو:فائل لاہو\n",
      "permalink:   https://www.express.pk/story/2268048/16\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268048-cricketteam-1641480632-866-640x480.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       پی سی بی کا پاور ہٹنگ کوچ لانا زبردست فیصلہ ہے، فہیم اشرف\n",
      "description: پاور ہٹنگ کوچ آنے سے باؤنڈریز لگانے کی تعداد میں نمایاں اضافہ دیکھنے کو مل سکتا \n",
      "permalink:   https://www.express.pk/story/2267964/16\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2267964-faheemashrafx-1641468975-605-640x480.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       قومی کرکٹر عابد علی روبہ صحت، گھر جانے کی اجازت\n",
      "description: عابد علی کی تمام رپورٹس تسلی بخش ہیں، ڈاکٹرز ( فائل فوٹو انٹرنیٹ) کراچی:قومی کرک\n",
      "permalink:   https://www.express.pk/story/2268029/16\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268029-abidali-1641478509-562-640x480.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       پیٹرول قیمتوں پر احتجاج: قازق صدر کا فوج کو مظاہرین پر گولیاں برسانے کا حکم\n",
      "description: جھڑپوں میں 18 پولیس اہلکار اور 28 شہری ہلاک ہوچکے ہیں، فوٹو: فائل نورسلطان:قازقس\n",
      "permalink:   https://www.express.pk/story/2268329/10\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268329-kazkhistan-1641551792-862-640x480.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       ایتھوپیا کے پناہ گزین کیمپ پر میزائل حملے میں بچوں سمیت 3 افراد ہلاک\n",
      "description: پناہ گزینوں کا تعلق ایریٹیریا سے ہے، فوٹو: فائل ادیس ابابا:ایتھوپیا کے شورش زدہ \n",
      "permalink:   https://www.express.pk/story/2268394/10\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268394-airraidkilledariterianrefugees-1641555693-525-640x480.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       فلسطین میں ہولناک ٹریفک حادثے میں 8 افراد جاں بحق، ویڈیو وائرل\n",
      "description: حادثے میں دو افراد شدید زخمی بھی ہوئے، فوٹو: ٹوئٹر غزہ:مغربی کنارے میں مسافر وین\n",
      "permalink:   https://www.express.pk/story/2268347/10\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268347-westbankaccidentkilledpalestanian-1641550717-698-640x480.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       ایران میں مسافر طیارے کی سڑک پرلینڈنگ، 5 افراد زخمی\n",
      "description: ایمرجنسی لینڈنگ کرنے والے طیارے میں 119مسافرسوارتھے:فوٹو:فائل تہران:ایران میں طی\n",
      "permalink:   https://www.express.pk/story/2268349/10\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268349-iranemergencylanding-1641549833-798-640x480.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       مقبوضہ کشمیرمیں بھارت کی ریاستی دہشتگردی،3نوجوان شہید\n",
      "description: بڈگام اورقریبی علاقوں کا محاصرہ جاری ہے:فوٹو:فائل سرینگر:مقبوضہ کشمیرمیں بھارتی \n",
      "permalink:   https://www.express.pk/story/2268305/10\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268305-iokfakeencounter-1641542836-462-640x480.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       جنوبی کوریا کے صدارتی انتخاب میں گنج پن کا علاج اہم مسئلہ بن گیا\n",
      "description: بالوں سے محروم افراد صدرجینگ کے حق میں مہم چلا رہے ہیں:فوٹو:فائل سئیول:جنوبی کور\n",
      "permalink:   https://www.express.pk/story/2268257/10\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268257-southkoreapresident-1641535348-654-640x480.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       امریکا میں انسانوں سے مماثل روبوٹ کی نمائش\n",
      "description: چہرے پر انسان جیسے جذبات و تاثرات سجائے ہوئے امیکا روبوٹ۔ (فوٹو: انجینئرڈ آرٹس)\n",
      "permalink:   https://www.express.pk/story/2268154/508\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268154-untitledcopy-1641498066-421-640x480.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       لاتعداد تفریحی کام کرنے والا سام سنگ فری اسٹائل پروجیکٹر\n",
      "description: سام سنگ نے ہرفن مولا ہلکا پھلکا ایچ ڈی ویڈیو پروجیکٹر پیش کردیا (فوٹو: فائل) جنو\n",
      "permalink:   https://www.express.pk/story/2267626/508\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2267626-sam-1641394420-854-640x480.gif\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       پرائیویسی کی خلاف ورزی پر گوگل اور فیس بُک پر کروڑوں روپے جرمانہ عائد\n",
      "description: [فائل-فوٹو] پیرس:فرانس نے فیس بُک اور گوگل پر آن لائن ٹریکنگ سے بچنے کیلئے صارف\n",
      "permalink:   https://www.express.pk/story/2268019/508\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268019-untitledcopy-1641478581-744-640x480.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       روس اور چین مل کر چاند پر اسٹیشن تعمیر کریں گے\n",
      "description: چاند پر روس اور چین کے مشترکہ اڈے کی خیالی تصویر۔ (بحوالہ: سی این ایس اے) ماسکو \n",
      "permalink:   https://www.express.pk/story/2267180/508\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2267180-russiachinalunarbasex-1641297666-867-640x480.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       ایپل 3,000 ارب ڈالر مالیت کی کمپنی بن گئی\n",
      "description: ایپل کمپنی اس وقت دنیا کی پہلی کمپنی بن گئی ہے جس نے تین ٹریلیئن ڈالر کے اثاثےبن\n",
      "permalink:   https://www.express.pk/story/2267267/508\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2267267-apple-1641307014-248-640x480.gif\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       موبائل سے کھلنے اور چلنے والی دنیا کی پہلی گاڑی\n",
      "description: اے آئی ٹی او نامی ایف فائیو اسمارٹ کار متعارف کرائی گئی ہے۔ (تصویر ہواوے) بیجنگ\n",
      "permalink:   https://www.express.pk/story/2267286/508\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2267286-huwawei-1641312407-849-640x480.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          25\n",
      "title:       سندھ کے نجی اسکولوں میں 12 تا 18 سال تک کے طلباء اور اسٹاف کی ویکسینیشن لازمی قرار\n",
      "description: ویکسینیشن نہ کروانے والے اسکولوں کے خلاف کارروائی کی جائے گی، نوٹیفکیشن جاری ۔ ف\n",
      "permalink:   https://www.express.pk/story/2268407/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268407-school-1641557760-806-640x480.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          26\n",
      "title:       رانا شمیم بیان حلفی کیس؛ ملزمان پر توہین عدالت کی فرد جرم عائد کرنے کی کارروائی موخر\n",
      "description: اٹارنی جنرل نے بھی صحافیوں کے خلاف فرد جرم عائد کرنے کی کارروائی ملتوی کرنے کی ت\n",
      "permalink:   https://www.express.pk/story/2268278/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268278-ranashamim-1641538959-734-640x480.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          27\n",
      "title:       نتھیا گلی میں ہوٹل کا ٹھیکہ ممتاز مسلم کو قانون کے مطابق دیا گیا، گلیات اتھارٹی\n",
      "description: میڈیا پر نشر ہونے والی خبریں من گھڑت، بے بنیاد اور حقائق کے خلاف ہیں، ترجمان گلی\n",
      "permalink:   https://www.express.pk/story/2268352/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268352-mumtazmuslim-1641550136-380-640x480.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          28\n",
      "title:       وجیہہ سواتی کی میت امریکا روانہ کردی گئی\n",
      "description: وجیہہ سواتی قتل کیس میں سابق شوہرسمیت6 افراد کوگرفتارکیا گیا ہے:فوٹو:فائل راولپن\n",
      "permalink:   https://www.express.pk/story/2268300/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268300-wajihasawati-1641541747-633-640x480.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          29\n",
      "title:       دوسری شادی نہ ہونے پر فائرنگ کرنے والا ملزم اپنی 3 بیٹیوں کا بھی قاتل نکلا\n",
      "description: ملزم نےاندھا دھند فائرنگ کرکے ایک شخص کو قتل جبکہ دوخواتین سمیت پانچ افراد کو شد\n",
      "permalink:   https://www.express.pk/story/2268340/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268340-bodies-1641550424-767-640x480.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          30\n",
      "title:       اسلام آباد ایئرپورٹ پر مسافر سے کروڑوں روپے مالیت کی ہیروئن برآمد\n",
      "description: سوات کا رہائشی تازہ خان نامی مسافر سعودی عرب جارہا تھا، اے این ایف ۔ فوٹو:فائل ا\n",
      "permalink:   https://www.express.pk/story/2268253/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268253-drugs-1641535919-989-640x480.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          31\n",
      "title:       دُرود اس پر کہ جس کے خُلق کی تفسیر قرآں ہے\n",
      "description: یہ جہاں چیز ہے کیا، لوح و قلم تیرے ہیںفوٹو : فائل خدائے عزوجل نے تخلیق کائنات کے\n",
      "permalink:   https://www.express.pk/story/2268206/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268206-masjidenabwi-1641500991-426-640x480.jpg\n",
      "category:    Islam\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          32\n",
      "title:       وقت کی قدر و قیمت جانیے۔۔۔۔!\n",
      "description: فوٹو : فائل  پروردگارِ عالم کی ذاتِ پاک کی عطا کردہ زندگی میں، ساعتیں، لمحے، گھڑ\n",
      "permalink:   https://www.express.pk/story/2268209/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2022/01/2268209-watchfilex-1641501103-934-640x480.jpg\n",
      "category:    Islam\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          33\n",
      "title:       مسلم ریاست میں غیر مسلم کے حقوق\n",
      "description: ۔ فوٹو: فائل اﷲ تعالیٰ کا پسندیدہ دین اسلام، دینِ امن ہے اور یہ معاشرے میں رہنے \n",
      "permalink:   https://www.express.pk/story/2265349/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2021/12/2265349-namazx-1640897624-720-640x480.jpg\n",
      "category:    Islam\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          34\n",
      "title:       ناپ تول کا اسلامی اصول\n",
      "description: فوٹو : فائل تجارت اور سوداگری کے باہمی لین دین کے معاملات میں کمی بیشی کرنے یعنی\n",
      "permalink:   https://www.express.pk/story/2265357/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2021/12/2265357-adalx-1640897919-951-640x480.jpg\n",
      "category:    Islam\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          35\n",
      "title:       کامیاب زندگی کی شاہ کلید\n",
      "description: مومن کی پہچان یہ ہے نعمتوں پر شکر بجا لاتا اور تکالیف و آزمائشوں پر صبر سے کام \n",
      "permalink:   https://www.express.pk/story/2262700/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2021/12/2262700-namazjumacoronavirusx-1640293357-764-640x480.jpg\n",
      "category:    Islam\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          36\n",
      "title:       اپنا محاسبہ کیجیے۔۔۔۔!\n",
      "description: سال مہینوں کی طرح اور مہینے دنوں کی طرح اور دن گھنٹوں کی طرح گزرتے چلے جارہے ہیں\n",
      "permalink:   https://www.express.pk/story/2259738/1\n",
      "channel:     Express\n",
      "image:       https://c.express.pk/2021/12/2259738-namazx-1639687541-484-640x480.jpg\n",
      "category:    Islam\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 36\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources5 = [[\"https://www.express.pk/health/\", \"Health\", \"Express\"],\n",
    "                [\"https://www.express.pk/sports/\", \"Sports\", \"Express\"],\n",
    "                [\"https://www.express.pk/world/\", \"International\", \"Express\"],\n",
    "                [\"https://www.express.pk/science/\", \"IT-Science\", \"Express\"],\n",
    "                [\"https://www.express.pk/pakistan/\", \"National-Local\", \"Express\"],\n",
    "                [\"https://www.express.pk/religion/\", \"Islam\", \"Express\"]\n",
    "               ]\n",
    "    if Express == True:\n",
    "        print('Getting Express News...')\n",
    "        counter = 0\n",
    "        i = 1\n",
    "        permalinkArrayExpressNews = []\n",
    "        for source in sources5:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='cstoreyitem'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = div.div.a['href']\n",
    "                        permalinkArrayExpressNews.append([permalink,source[1],source[2]])\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayExpressNews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='primary'):\n",
    "                try:\n",
    "                    detail_title = div.div.h1.text.replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.div.find('div',class_='span-10').div.div.img['src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.div.find('div',class_='span-16 story-cont').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text.replace('\\n','').replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='Urdu'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': permalink,\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting SAMA News...\n",
      "total news found on listting page : 16\n",
      "\n",
      "No:          1\n",
      "title:       What not to miss at 16th Karachi International Book Fair\n",
      "description: Photo: SAMAA DigitalIf you want to see what a diverse Karachi has incommon, it i\n",
      "permalink:   https://www.samaa.tv/lifeandstyle/2022/01/what-not-to-miss-at-16th-karachi-international-book-fair/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2022/01/Untitled-1-2.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       Naseeruddin Shah calls Mughals ‘refugees,’ warns of Indian ‘civil war’\n",
      "description:  Naseeruddin Shah is coming under fire for making comments on “civil war in Indi\n",
      "permalink:   https://www.samaa.tv/lifeandstyle/2021/12/naseeruddin-shah-calls-mughals-refugees-warns-of-indian-civil-war/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2021/06/768042-749637-728395-naseeruddin-shah-090818.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       British socialite Ghislaine Maxwell convicted of sex trafficking\n",
      "description: Photo: AFP British socialite Ghislaine Maxwell was found guilty in New York on W\n",
      "permalink:   https://www.samaa.tv/lifeandstyle/2021/12/british-socialite-ghislaine-maxwell-convicted-of-sex-trafficking/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2021/12/Ghislaine-Maxwell.png\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       In pictures: Christmas celebrations light up Pakistan\n",
      "description: Christian devotees pray in a church on Christmas Eve in Quetta on December 24, 2\n",
      "permalink:   https://www.samaa.tv/lifeandstyle/2021/12/in-pictures-christmas-preparations-in-full-swing-across-pakistan/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2021/12/000_9V86DH.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       Pakistan team to get booster shots today: report\n",
      "description: Photo: AFPThe Pakistan CricketBoard (PCB) has reportedly decided to give a boost\n",
      "permalink:   https://www.samaa.tv/sports/cricket/2022/01/pakistan-team-to-get-booster-shots-today-report/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2022/01/Pakistan-Team2.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Hashim Amla in awe of ‘magician’ Mohammad Asif\n",
      "description: Photo: Screenshot-YouTube Former South Africabatter Hashim Amla has heaped prais\n",
      "permalink:   https://www.samaa.tv/sports/cricket/2022/01/hashim-amla-in-awe-of-magician-mohammad-asif/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2022/01/Asif-and-Amla.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       Rizwan steals limelight in PCB Awards 2021\n",
      "description: Photo: PCBWicketkeeper Mohammad Rizwan has bagged the PCB’s Most Valuable Cricke\n",
      "permalink:   https://www.samaa.tv/sports/cricket/2022/01/live-pcb-awards-2021/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2022/01/M-rizwan-1.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       Khawaja back to haunt England with Sydney ton as Australia dominate\n",
      "description: Photo: AFPUsman Khawajareturned to haunt England Thursday with a stylish century\n",
      "permalink:   https://www.samaa.tv/sports/cricket/2022/01/khawaja-back-to-haunt-england-with-sydney-ton-as-australia-dominate/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2022/01/Khawaja.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Senate committee rejects increase in GST on formula milk, jewelry\n",
      "description: Photo: AFP The Senate Committee on Finance has strongly rejected the increase in\n",
      "permalink:   https://www.samaa.tv/news/2022/01/senate-committee-rejects-increase-in-gst-on-formula-milk-jewelry/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2020/03/Child-Milk.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       Court wants Islamabad’s Navy Sailing Club demolished\n",
      "description: Navy Sailing Club in Islamabad. The Islamabad High Court has ordered the demolit\n",
      "permalink:   https://www.samaa.tv/news/2022/01/court-wants-islamabads-navy-sailing-club-demolished/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2022/01/WhatsApp-Image-2022-01-07-at-10.59.55-AM.jpeg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       Omicron’s threat, Madina Masjid protest, Meesha Shafi case\n",
      "description: These are the stories, developments and updates we are following Friday, January\n",
      "permalink:   https://www.samaa.tv/news/2022/01/omicrons-threat-madina-masjid-protest-meesha-shafi-case/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2021/12/Todays-outlook_Updated_us-omicron-evms-in-pubnjab.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       Heavy rain, snow bring life to standstill in Balochistan\n",
      "description: Photo: AFP Nearly three consecutive days of heavy rain and snowfall have halted \n",
      "permalink:   https://www.samaa.tv/news/2022/01/heavy-rain-snow-bring-life-to-standstill-in-balochistan/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2022/01/balochistan-snow.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       Smartwatches: the next big thing or just a fad?\n",
      "description: The smartphone has become a necessary gadget for all generations in today’s tech\n",
      "permalink:   https://www.samaa.tv/video/2022/01/smartwatches-the-next-big-thing-or-just-a-fad/\n",
      "channel:     SAMAA\n",
      "image:       \n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       Get Creative with your New Camera Phone – TECNO Camon 18 Series\n",
      "description: Withthe new year right around the corner, many people are looking to upgrade the\n",
      "permalink:   https://www.samaa.tv/technology/2021/12/get-creative-with-your-new-camera-phone-tecno-camon-18-series/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2021/12/Tecno-Camon-18-series-1.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       Indian iPhone plant ‘on probation’ after mass food poisoning\n",
      "description: Foxconn's iPhone factory in the southern Indian state of Tamil Nadu employs some\n",
      "permalink:   https://www.samaa.tv/technology/2021/12/apple-puts-india-iphone-plant-on-probation-after-mass-food-poisoning/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2021/12/Foxconn-India-iPhone-plant-AFP.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       Tech 2022 trends: Meatless meat, Web 3.0, Big Tech battles\n",
      "description: Source: sig.orgAfter a year that made the terms WFH (work from home) and metaver\n",
      "permalink:   https://www.samaa.tv/technology/2021/12/tech-2022-trends-meatless-meat-web-3-0-big-tech-battles/\n",
      "channel:     SAMAA\n",
      "image:       https://www.samaa.tv/wp-content/uploads/2021/12/WhatsApp-Image-2021-12-29-at-12.01.00.jpeg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 16\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources6 = [\n",
    "                [\"https://www.samaa.tv/lifeandstyle/\", \"National-Local\", \"SAMAA\"],\n",
    "                [\"https://www.samaa.tv/sports/\", \"Sports\", \"SAMAA\"], \n",
    "                [\"https://www.samaa.tv/news/\", \"National-Local\", \"SAMAA\"],\n",
    "                [\"https://www.samaa.tv/technology/\", \"IT-Science\", \"SAMAA\"]\n",
    "                ]\n",
    "    if Sama == True:\n",
    "        print('Getting SAMA News...')\n",
    "        counter = 0\n",
    "        i = 1\n",
    "        permalinkArraySAMANews = []\n",
    "        for source in sources6:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='categorycolumn'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = div.find('div',class_='row').find('div',class_='respnb1').a['href']\n",
    "                        permalinkArraySAMANews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArraySAMANews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='col-md-8'):\n",
    "                try:\n",
    "                    detail_title = div.div.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = ''\n",
    "                    high_resolution_img_parent = div.find('div',class_='detail-10 detailnews').center\n",
    "                    high_resolution_img_text = high_resolution_img_parent.find(text=lambda text:isinstance(text, Comment))\n",
    "                    high_resolution_img_soup = BeautifulSoup(high_resolution_img_text , 'lxml')\n",
    "                    high_resolution_image = high_resolution_img_soup.img['src']\n",
    "                except:\n",
    "                    high_resolution_img = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='detailnews').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text.replace('\\n','').replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1   \n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Duniya News...\n",
      "total news found on listting page : 0\n",
      "\n",
      "No:          1\n",
      "title:       Billionaire's looted art still on display at Israel Museum\n",
      "description: JERUSALEM (AP) — One of the Israel Museum’s biggest patrons, American billionair\n",
      "permalink:   https://dunyanews.tv/index.php/en/World/635891-Billionaire-looted-art-still-on-display-at-Israel-Museum\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635891_99571632.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       As cases rise, India fears another Covid catastrophe\n",
      "description: NEW DELHI (AFP) - Haunted by the spectre of last year’s crisis, India is bracing\n",
      "permalink:   https://dunyanews.tv/index.php/en/World/635889-As-cases-rise-India-fears-another-Covid-catastrophe\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635889_34531222.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       As cases rise, India fears another Covid catastrophe\n",
      "description: NEW DELHI (AFP) - Haunted by the spectre of last years crisis, India is bracing \n",
      "permalink:   https://dunyanews.tv/index.php/en/World/635888-As-cases-rise,-India-fears-another-Covid-catastrophe\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635888_65024456.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       UK to deploy troops to virus-hit London hospitals\n",
      "description: LONDON (AFP) - Britain will deploy troops to hospitals in London to alleviate se\n",
      "permalink:   https://dunyanews.tv/index.php/en/World/635887-UK-to-deploy-troops-to-virus-hit-London-hospitals\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635887_47606017.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       Clashes in largest Kazakh city, Moscow-led troops arrive\n",
      "description: ALMATY (AFP) - Bursts of gunfire echoed through the streets of Kazakhstans large\n",
      "permalink:   https://dunyanews.tv/index.php/en/World/635843-Clashes-in-largest-Kazakh-city,-Moscow-led-troops-arrive\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635843_67816346.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Tehran to Seoul: Don't wait to release frozen funds\n",
      "description: TEHRAN (AFP) - Iranian negotiator Ali Bagheri on Thursday urged South Korea to r\n",
      "permalink:   https://dunyanews.tv/index.php/en/World/635841-Tehran-to-Seoul:-Don\\'t-wait-to-release-frozen-funds\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635841_69713933.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       Babar's father praises son for winning \"Cricketer of the Year\" award\n",
      "description: LAHORE (Dunya News) – Pakistan Cricket team’s captain Babar Azam received praise\n",
      "permalink:   https://dunyanews.tv/index.php/en/Cricket/635892-Babars-father-praises-son-for-winning-Cricketer-of-the-Year-award\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635892_52410094.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       Bairstow hits ton, Stokes defies pain as England fight back\n",
      "description: SYDNEY, (AFP) - Jonny Bairstow broke through for his first Test hundred in Austr\n",
      "permalink:   https://dunyanews.tv/index.php/en/Cricket/635877-Bairstow-hits-ton-Stokes-defies-pain-England-fight-back\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635877_28865959.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Dean Elgar takes the blows to inspire South African win over India\n",
      "description: JOHANNESBURG (AFP) - South African captain Dean Elgar said there was a double si\n",
      "permalink:   https://dunyanews.tv/index.php/en/Cricket/635821-Dean-Elgar-takes-the-blows-to-inspire-South-African-win-over-India\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635821_42807577.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       Elgar steers South Africa to victory over India in second Test\n",
      "description: JOHANNESBURG (AFP) - South Africa captain Dean Elgar hit a superb 96 not out to \n",
      "permalink:   https://dunyanews.tv/index.php/en/Cricket/635819-Elgar-steers-South-Africa-to-victory-over-India-in-second-Test\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-06-22/news_big_images/635819_59189173.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       India batting unit need some sharpening, says coach Dravid\n",
      "description: JOHANNESBURG (Reuters) - India’s batsmen need to better seize their chances and \n",
      "permalink:   https://dunyanews.tv/index.php/en/Cricket/635810-India-batting-unit-need-some-sharpening-says-coach-Dravid\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-06-22/news_big_images/635810_66131196.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       Women's Ashes brought forward due to World Cup quarantine rules\n",
      "description: (Reuters) - The Women’s Ashes series has been brought forward by a week to allow\n",
      "permalink:   https://dunyanews.tv/index.php/en/Cricket/635801-Women-Ashes-brought-forward-due-to-World-Cup-quarantine-rules\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-06-22/news_big_images/635801_94312401.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       Int'l community's assistance to Afghanistan imperative for regional peace: COAS\n",
      "description: RAWALPINDI (Dunya News) – Chief of Army Staff General Qamar Javed Bajwa said on \n",
      "permalink:   https://dunyanews.tv/index.php/en/Pakistan/635902-Int-community-assistance-Afghanistan-imperative-regional-peace-COAS-ISPR\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635902_52132656.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       VIDEO: Jahangir Tarin responds to question regarding tilt towards PML-N\n",
      "description: LAHORE (Dunya News) – Pakistan Tehreek-e-Insaf (PTI) leader Jahangir Khan Tareen\n",
      "permalink:   https://dunyanews.tv/index.php/en/Pakistan/635900-Jahaz-kisi-taraf-bhi-ja-sakta-hai-Jahangir-Tarin-responds-question-PML-N\n",
      "channel:     Dunya\n",
      "image:       \n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       Bad weather forces PIA to divert three Lahore-bound flights to Islamabad\n",
      "description: ISLAMABAD (APP) - Pakistan International Airlines (PIA) diverted its three domes\n",
      "permalink:   https://dunyanews.tv/index.php/en/Pakistan/635884-Bad-weather-forces-PIA-to-divert-three-Lahore-bound-flights-to-IIAP\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635884_63265143.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       Three children killed in roof collapse near Lesbela\n",
      "description: LASBELA (Dunya News) – Three children were killed while two other people sustain\n",
      "permalink:   https://dunyanews.tv/index.php/en/Pakistan/635882-Three-children-killed-in-roof-collapse-near-Lesbela\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635882_65459716.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       PM seeks report from Balochistan CM on damages occurred due to recent rains\n",
      "description: ISLAMABAD (Dunya News) - Prime Minister Imran Khan has sought report from Chief \n",
      "permalink:   https://dunyanews.tv/index.php/en/Pakistan/635881-PM-seeks-report-from-Balochistan-CM-on-damages-occurred-due-to-recent-ra\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635881_82173455.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       IHC again defers indictment of Rana Shamim, others\n",
      "description: ISLAMABAD (Dunya News) - The Islamabad High Court (IHC) on Friday deferred indic\n",
      "permalink:   https://dunyanews.tv/index.php/en/Pakistan/635879-IHC-again-defers-indictment-of-Rana-Shamim-others\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635879_52520157.jpg\n",
      "category:    National-Local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       Japan tycoon Maezawa returns from space with business dreams\n",
      "description: TOKYO (AP) — “Space now,” was what Japanese billionaire Yusaku Maezawa wanted to\n",
      "permalink:   https://dunyanews.tv/index.php/en/Technology/635893-Japan-tycoon-Maezawa-returns-from-space-with-business-dreams\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635893_22436000.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       France hits Google, Facebook with huge fines over 'cookies'\n",
      "description: PARIS (AFP) - French regulators have hit Google and Facebook with 210 million eu\n",
      "permalink:   https://dunyanews.tv/index.php/en/Technology/635853-France-hits-Google-Facebook-with-huge-fines-over-cookies\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635853_67047031.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       Samsung Electronics forecasts 52.5% jump in Q4 profits on record sales\n",
      "description: SEOUL (AFP) - Samsung Electronics expects operating profits for the fourth quart\n",
      "permalink:   https://dunyanews.tv/index.php/en/Technology/635851-Samsung-Electronics-forecasts-52-5-jump-in-Q4-profits-on-record-sales\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635851_82184980.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       Ammonia and paper: Sustainability ideas at CES tech show\n",
      "description: LAS VEGAS (AFP) - The CES tech show has its share of gadgets that may be clutter\n",
      "permalink:   https://dunyanews.tv/index.php/en/Technology/635850-Ammonia-and-paper-Sustainability-ideas-at-CES-tech-show\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635850_51472156.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       BMW's new iX Flow, a car with color changing feature\n",
      "description: (Web Desk): At the annual Consumer Electronics Show trade fair BMW showed off th\n",
      "permalink:   https://dunyanews.tv/index.php/en/Technology/635805-BMW-new-iX-Flow-car-color-changing-feature\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-06-22/news_big_images/635805_22218951.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       Robot tractors may be heading to some US farms\n",
      "description: LAS VEGAS (AFP) - Driverless plows and autonomous tools to weed vegetable plots \n",
      "permalink:   https://dunyanews.tv/index.php/en/Technology/635721-Robot-tractors-may-be-heading-to-some-US-farms\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-06-22/news_big_images/635721_83666186.jpg\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          25\n",
      "title:       Khusro Bakhtiar rules out shortage of urea in country\n",
      "description: ISLAMABAD (Dunya News) - Minister for Industries and Production Khusro Bakhtiar \n",
      "permalink:   https://dunyanews.tv/index.php/en/Business/635883-Khusro-Bakhtiar-rules-out-shortage-of-urea-in-country\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635883_13169382.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          26\n",
      "title:       Govt taking steps to bring improvement in manufacturing sector: Khusro\n",
      "description: ISLAMABAD (Web Desk) - Federal Minister for Industries and Production, Makhdum K\n",
      "permalink:   https://dunyanews.tv/index.php/en/Business/635874-Govt-taking-steps-to-bring-improvement-in-manufacturing-sector-Khusro\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635874_23170986.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          27\n",
      "title:       Asian markets mixed as attention turns to US jobs\n",
      "description: HONG KONG (AFP) - Asian equities were mixed Friday as investors battled to insti\n",
      "permalink:   https://dunyanews.tv/index.php/en/Business/635847-Asian-markets-mixed-as-attention-turns-to-US-jobs\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-07-22/news_big_images/635847_86445175.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          28\n",
      "title:       Asad reaffirms govt's commitment to reduce gender disparity\n",
      "description: ISLAMABAD (Dunya News) – Minister for Planning Development and Special Initiativ\n",
      "permalink:   https://dunyanews.tv/index.php/en/Business/635813-Asad-reaffirms-govt-commitment-to-reduce-gender-disparity\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-06-22/news_big_images/635813_82801487.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          29\n",
      "title:       PSX gains 325.59 points to close at 45,082.30 points\n",
      "description: ISLAMABAD (Dunya News) – The KSE 100-index of the Pakistan Stock Exchange (PSX) \n",
      "permalink:   https://dunyanews.tv/index.php/en/Business/635811-PSX-gains-325.59-points-to-close-at-45,082.30-points-\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-06-22/news_big_images/635811_17603380.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          30\n",
      "title:       Europe stocks sink after Fed signals faster rate hikes\n",
      "description: LONDON (AFP) - European equities sank Thursday at the open after the Federal Res\n",
      "permalink:   https://dunyanews.tv/index.php/en/Business/635768-Europe-stocks-sink-after-Fed-signals-faster-rate-hikes\n",
      "channel:     Dunya\n",
      "image:       https://img.dunyanews.tv/news/2022/January/01-06-22/news_big_images/635768_58282183.jpg\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 30\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources7 = [\n",
    "                [\"https://dunyanews.tv/en/World\", \"International\", \"Dunya\"],\n",
    "                [\"https://dunyanews.tv/en/Cricket\", \"Sports\", \"Dunya\"], \n",
    "                [\"https://dunyanews.tv/en/Pakistan\", \"National-Local\", \"Dunya\"],\n",
    "                [\"https://dunyanews.tv/en/Technology\", \"IT-Science\", \"Dunya\"],\n",
    "                [\"https://dunyanews.tv/en/Business\",\"Business\",\"Dunya\"]\n",
    "               ]\n",
    "    if Duniya == True:\n",
    "        counter = 0\n",
    "        i = 1\n",
    "        print('Getting Duniya News...')\n",
    "        permalinkArrayDuniyaNews = []\n",
    "        for source in sources7: \n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('article'):\n",
    "                if i <= 6:\n",
    "                    if article.div.a['href']:\n",
    "                        try:\n",
    "                            permalink = 'https://dunyanews.tv' + article.div.a['href']\n",
    "                            permalinkArrayDuniyaNews.append([permalink,source[1],source[2]])\n",
    "                        except:\n",
    "                            permalink = ''\n",
    "                    else:\n",
    "                        try:\n",
    "                            permalink = 'https://dunyanews.tv' + article.h3.a['href']\n",
    "                            permalinkArrayDuniyaNews.append([permalink,source[1],source[2]])\n",
    "                        except:\n",
    "                            permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayDuniyaNews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "\n",
    "            for div in soup.find_all('div',{'id':'main-heading'}):\n",
    "\n",
    "                try:\n",
    "                    detail_title = div.find('div',class_='date').h2.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.find('div',class_='date').find('div',{'id':'post_content'}).article.find('div',class_='pic').img['src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='date').find('div',{'id':'post_content'}).article.find('div',class_='post_content').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text.replace('\\n','').replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': permalink,\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Bol News...\n",
      "total news found on listting page : 36\n",
      "\n",
      "No:          1\n",
      "title:       \n",
      "description: MELBOURNE: The rapidly spreading Omicron variant, no Serena Williams or Roger Fe\n",
      "permalink:   https://www.bolnews.com/sports/2022/01/australian-open-threatening-to-become-not-so-happy-slam/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       \n",
      "description: YAOUNDÉ: The Africa Cup of Nations begins in Cameroon on Sunday with the contine\n",
      "permalink:   https://www.bolnews.com/sports/2022/01/mane-salah-and-more-six-stars-to-watch-at-the-cup-of-nations/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       \n",
      "description: CHRISTCHURCH: Ross Taylor saluted Bangladesh for their upset win in the first Te\n",
      "permalink:   https://www.bolnews.com/sports/2022/01/n-zealand-great-taylor-wants-winning-farewell-after-shock-loss/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       \n",
      "description: SYDNEY: Wincing Ben Stokes produced a counter-punching half-century to help Engl\n",
      "permalink:   https://www.bolnews.com/sports/2022/01/stokes-defies-injury-with-fighting-half-century-in-ashes-test/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       \n",
      "description: Abid Ali, the Pakistan cricket team’s Test opener, received permission to travel\n",
      "permalink:   https://www.bolnews.com/sports/2022/01/abid-ali-has-been-granted-permission-to-travel-from-karachi-to-lahore/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       \n",
      "description: LAHORE: Khyber Pakhtunkhwa’s Abbas Ali has replaced Abdul Wahid Bangalzai in the\n",
      "permalink:   https://www.bolnews.com/sports/2022/01/abbas-ali-replaces-abdul-wahid-in-pakistan-u19-squad/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       \n",
      "description: CAIRO, Jan. 7 (Xinhua) — Egypt seeks to learn from China’s “unique, distinguishe\n",
      "permalink:   https://www.bolnews.com/international/2022/01/interview-egypt-seeks-to-learn-from-chinas-unique-development-experience-official/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       \n",
      "description: TAIPEI, Jan. 7 (Xinhua) — Taiwan reported four new locally-transmitted confirmed\n",
      "permalink:   https://www.bolnews.com/health/2022/01/taiwan-airport-cluster-infections-rise-to-12/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       \n",
      "description: Sandra Bullock recently praised streaming services for fostering greater diversi\n",
      "permalink:   https://www.bolnews.com/amazing/2022/01/sandra-bullocks-were-all-praises-for-streaming-sites/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       \n",
      "description: Experts have provided light on Prince William’s knee-jerk reaction to Prince Har\n",
      "permalink:   https://www.bolnews.com/entertainment/2022/01/prince-harry-and-prince-william-head-to-head-once-again/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       \n",
      "description: The seventh international seminar of the Organisation of Islamic Cooperation (OI\n",
      "permalink:   https://www.bolnews.com/latest/2022/01/oic-seminar-urges-states-to-introduce-strong-anti-corruption-legislations/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       \n",
      "description: AR Rehman is one of the highly recognized and loved singers in India. But, only \n",
      "permalink:   https://www.bolnews.com/bollywood/2022/01/the-song-highway-is-the-favorite-of-ar-rehman/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       \n",
      "description: HONG KONG, Jan. 6 (Xinhua) — Hong Kong on Thursday reported 33 new cases of COVI\n",
      "permalink:   https://www.bolnews.com/health/2022/01/hong-kong-reports-33-new-covid-19-cases/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       \n",
      "description: BERLIN, Jan. 6 (Xinhua) — Daily COVID-19 infections in Germany surged by 64,340 \n",
      "permalink:   https://www.bolnews.com/health/2022/01/germany-reports-64340-new-covid-19-cases-as-infections-continue-to-surge/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       \n",
      "description: MANILA, Jan 6, 2022 (AFP) –Philippine President Rodrigo Duterte on Thursday orde\n",
      "permalink:   https://www.bolnews.com/health/2022/01/arrest-unvaccinated-if-they-go-outdoors-philippines-duterte/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       \n",
      "description: NEW DELHI, Jan. 6 (Xinhua) — Indian officials on Thursday said that 125 passenge\n",
      "permalink:   https://www.bolnews.com/health/2022/01/125-air-passengers-test-positive-for-covid-19-upon-arrival-in-the-northern-indian-city/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       \n",
      "description: Lilly Singh, an Indo-Canadian comedian, tested positive for COVID-19. Lilly Sing\n",
      "permalink:   https://www.bolnews.com/entertainment/2022/01/indo-canadian-youtuber-lily-singh-tests-positive-for-covid-19/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       \n",
      "description: PARIS, Jan 6, 2022 (AFP) –A series of coronavirus outbreaks have cropped up on c\n",
      "permalink:   https://www.bolnews.com/health/2022/01/omicron-crashes-cruise-party/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       \n",
      "description: Britain’s favorite Adele is back to winning hearts with a small preview of her n\n",
      "permalink:   https://www.bolnews.com/amazing/2022/01/adele-gives-a-sneakpeak-on-her-new-single-oh-my-god/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       \n",
      "description: Actors Hiba Bukhari and Arez Ahmed undoubtedly make a beautiful pair after they \n",
      "permalink:   https://www.bolnews.com/entertainment/2022/01/watch-newlyweds-hiba-bukhari-arez-ahmed-beam-with-happiness/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       \n",
      "description: The gorgeous actress Sana Javed is giving her fans tourist vibes in the latest p\n",
      "permalink:   https://www.bolnews.com/entertainment/2022/01/sana-javed-flashes-a-pose-at-top-of-burj-khalifa/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       \n",
      "description: Sandra Bullock recently praised streaming services for fostering greater diversi\n",
      "permalink:   https://www.bolnews.com/amazing/2022/01/sandra-bullocks-were-all-praises-for-streaming-sites/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       \n",
      "description: What a comeback actress Anushka Sharma has made after years with Chakda Xpress t\n",
      "permalink:   https://www.bolnews.com/entertainment/2022/01/anushka-sharma-drops-the-first-look-of-chakda-xpress/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       \n",
      "description: Indian actress Koena Mitra has once again opened up about her surgery and shared\n",
      "permalink:   https://www.bolnews.com/bollywood/2022/01/koena-mitra-opens-up-about-hard-days-in-industry-after-plastic-surgery/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          25\n",
      "title:       \n",
      "description: After months of anticipation, Samsung unveiled the Galaxy S21 FE earlier this we\n",
      "permalink:   https://www.bolnews.com/technology/2022/01/samsung-galaxy-s21-fe-goes-on-sale/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          26\n",
      "title:       \n",
      "description: In Pakistan, the Nokia G100 costs Rs. 24,100. In US dollars, this Nokia new mode\n",
      "permalink:   https://www.bolnews.com/technology/2022/01/nokia-g100-price-in-pakistan-specifications-release-date/\n",
      "channel:     Bol\n",
      "image:       https://bolnews.s3.eu-central-1.amazonaws.com/wp-content/uploads/2022/01/hmd_global_unveils_nokia_c100_c200_g100_and_g400_ces_2022_social_0.jpg-1.webp\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          27\n",
      "title:       \n",
      "description: The Nokia C200 is likely to cost Rs. 23,999 in Pakistan. On April 30, 2022, Noki\n",
      "permalink:   https://www.bolnews.com/technology/2022/01/nokia-c200-price-in-pakistan-specifications-release-date/\n",
      "channel:     Bol\n",
      "image:       https://bolnews.s3.eu-central-1.amazonaws.com/wp-content/uploads/2022/01/nokia_c100_c200_g100_g400_2760_android_authority_1641464561972-1.webp\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          28\n",
      "title:       \n",
      "description: The Nokia C100 is likely to cost Rs. 18,999 in Pakistan. On April 30, 2022, Noki\n",
      "permalink:   https://www.bolnews.com/technology/2022/01/nokia-c100-price-in-pakistan-specifications-release-date/\n",
      "channel:     Bol\n",
      "image:       https://bolnews.s3.eu-central-1.amazonaws.com/wp-content/uploads/2022/01/nokia_c100_c200_g100_g400_2760_android_authority_1641464561972.webp\n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          29\n",
      "title:       \n",
      "description: If you’re looking to purchase a new iPhone in 2022, you’ll have a lot of alterna\n",
      "permalink:   https://www.bolnews.com/technology/2022/01/iphone-13-pro-vs-iphone-13-which-should-you-buy-in-2022/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          30\n",
      "title:       \n",
      "description: Google announced plans at CES 2022 to improve device connectivity for Android ph\n",
      "permalink:   https://www.bolnews.com/technology/2022/01/google-to-introduce-connectivity-features-for-smart-devices-in-2022/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    IT-Science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          31\n",
      "title:       \n",
      "description: KARACHI: According to the Sarafa market, today’sgold rates in karachi (Pakistan)\n",
      "permalink:   https://www.bolnews.com/business/2022/01/gold-rates-todays-gold-rate-in-karachi-on-7th-january-2022/\n",
      "channel:     Bol\n",
      "image:       https://bolnews.s3.eu-central-1.amazonaws.com/wp-content/uploads/2022/01/02161806/Gold-Rates-in-Pakistan.jpg.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          32\n",
      "title:       \n",
      "description: KARACHI: The average customer rates of buying and selling of US dollar (USD) aga\n",
      "permalink:   https://www.bolnews.com/business/2022/01/usd-to-pkr-open-market-exchange-rates-on-7th-january-2022/\n",
      "channel:     Bol\n",
      "image:       https://bolnews.s3.eu-central-1.amazonaws.com/wp-content/uploads/2022/01/1-8.jpg-1.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          33\n",
      "title:       \n",
      "description: DUBAI:Employees and schoolchildren juggled work and studies with prayers on the \n",
      "permalink:   https://www.bolnews.com/business/2022/01/uae-works-on-friday-for-first-time/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          34\n",
      "title:       \n",
      "description: KARACHI: The Power Cement Limited has signed an agreement to set up a 7MW solar \n",
      "permalink:   https://www.bolnews.com/business/2022/01/power-cement-burj-solar-energy-sign-deal-to-set-up-power-plant/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          35\n",
      "title:       \n",
      "description: KARACHI: The average customer rates of buying and selling of Canadian Dollar (CA\n",
      "permalink:   https://www.bolnews.com/business/2022/01/cad-to-pkr-todays-canadian-dollar-to-pkr-exchange-rates-on-7th-jan-2022/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          36\n",
      "title:       \n",
      "description: KARACHI: The average customer rates of buying and selling of Qatar Riyal (QAR) a\n",
      "permalink:   https://www.bolnews.com/business/2022/01/qar-to-pkr-todays-qatari-riyal-to-pkr-rates-on-7th-jan-2022/\n",
      "channel:     Bol\n",
      "image:       \n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 36\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources8 = [\n",
    "                [\"https://www.bolnews.com/sports\", \"Sports\", \"Bol\"],\n",
    "                [\"https://www.bolnews.com/international/\", \"International\", \"Bol\"],\n",
    "                [\"https://www.bolnews.com/health/\", \"Health\", \"Bol\"],\n",
    "                [\"https://www.bolnews.com/entertainment/\", \"Entertainment\", \"Bol\"],\n",
    "                [\"https://www.bolnews.com/technology/\", \"IT-Science\", \"Bol\"],\n",
    "                [\"https://www.bolnews.com/business/\",\"Business\",\"Bol\"]\n",
    "                ]\n",
    "    if Bol == True:\n",
    "        counter = 0\n",
    "        print('Getting Bol News...')\n",
    "        i = 1\n",
    "        permalinkArrayBolNews = []\n",
    "        for source in sources8:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='catesquarebox'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = div.a['href']\n",
    "                        permalinkArrayBolNews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayBolNews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for main in soup.find_all('main'):\n",
    "                try:\n",
    "                    detail_title = main.article.find('div',class_='headingsec').h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    if('https' in main.article.find('figure',class_='featuredimg').img['src']):\n",
    "                        high_resolution_image = main.article.find('figure',class_='featuredimg').img['src']\n",
    "\n",
    "                    else:\n",
    "                        high_resolution_image = 'https://bolnews.s3.amazonaws.com/' + main.find('article',class_='infinite-post active').find('figure',class_='featuredimg').img['src']\n",
    "\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = main.article.find('div',class_='content').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text.replace('\\n','').replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1     \n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Abbtakk News...\n",
      "total news found on listting page : 80\n",
      "\n",
      "No:          1\n",
      "title:       Wedding Festivities of Saboor Aly, Ali Ansari Begin\n",
      "description: Saboor Aly is all-set to say her wedding vows to Ali Ansari!The power couple is \n",
      "permalink:   https://abbtakk.tv/en/wedding-festivities-of-saboor-aly-ali-ansari-begin/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06210819/391846_6757464_updates.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       Emma Watson Post in Support of Palestinians Angers Israeli Envoys\n",
      "description: WEB DESK: The official Instagram account of British actress Emma Watson has expr\n",
      "permalink:   https://abbtakk.tv/en/emma-watson-post-in-support-of-palestinians-angers-israeli-envoys/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/05214258/ema.jpeg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       John Abraham Tests Positive For COVID-19\n",
      "description: John Abraham and his family are in quarantine after contracting COVID-19, Indian\n",
      "permalink:   https://abbtakk.tv/en/john-abraham-tests-positive-for-covid-19/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/03170605/391276_8097183_updates.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       Birth Anniversary Of Comedy King ‘Lehri’ Being Observed Today\n",
      "description: Karachi: The birth anniversary of comedy king Safeerullah Siddiqui‚ popularly kn\n",
      "permalink:   https://abbtakk.tv/en/birth-anniversary-of-comedy-king-lehri-being-observed-today/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.tv/en/wp-content/uploads/2017/09/Actor-Lehri.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       Teaser For Documentary Film ‘Karakoram Highway’ Released\n",
      "description: ISLAMABAD: Federal Minister Fawad Chaudary has shared the teaser for the documen\n",
      "permalink:   https://abbtakk.tv/en/teaser-for-documentary-film-karakoram-highway-released/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/31211519/634975_83183842.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Trailer of Sequel of “The Batman” Released\n",
      "description: After the trailer was released, we could see a new person playing the role of Ba\n",
      "permalink:   https://abbtakk.tv/en/trailer-of-sequel-of-the-batman-released/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/10/18214537/batman.jpeg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       ‘Spider-Man: No Way Home’ Rocks Box Office\n",
      "description: Spider-Man: No Way Home has now elevated to become the highest-grossing film of \n",
      "permalink:   https://abbtakk.tv/en/spider-man-no-way-home-rocks-box-office/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/27211200/spider-man.jpeg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       Trailer of Uncharted Released\n",
      "description: A brand new trailer for Uncharted has been released. The movie, that is. Co-star\n",
      "permalink:   https://abbtakk.tv/en/trailer-of-uncharted-released/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/24213417/ok.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Moin Akhtar 71 Birth Anniversary Celebrated Today\n",
      "description: The 71st birth anniversary of Pakistani entertainment icon Moin Akhtar was obser\n",
      "permalink:   https://abbtakk.tv/en/moin-akhtar-71-birth-anniversary-celebrated-today/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/24181842/4980978861640314860.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       Japanese Star Sayanka Kanda Dies\n",
      "description: Prominent Japanese actor and singer, Sayaka Kanda passed away at the age of 35 a\n",
      "permalink:   https://abbtakk.tv/en/japanese-star-sayanka-kanda-dies/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/23202944/moel.jpeg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       Fawad Announces Awards Worth Rs250mn For Artists\n",
      "description: ISLAMABAD: Minister for Information and Broadcasting Chaudhary Fawad Hussain has\n",
      "permalink:   https://abbtakk.tv/en/fawad-announces-awards-worth-rs250mn-for-artists/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/22213701/1640187730.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       Mahira Khan Turns 37 Today\n",
      "description: KARACHI: Pakistani superstar actress Mahira Khan is celebrating her 37th birthda\n",
      "permalink:   https://abbtakk.tv/en/mahira-khan-turns-37-today/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2020/10/12213933/mahira_620x330.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       Trailer of Hollywood Movie ‘Clean’ Released\n",
      "description: LOS ANGELES: The trailer of upcoming Hollywood thriller move Clean has been rele\n",
      "permalink:   https://abbtakk.tv/en/trailer-of-hollywood-movie-clean-released/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/20235707/Untitled-6.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       Salman Khan Gifts Range Rover Worth Rs30mn To Katrina Kaif\n",
      "description: Bollywood diva Katrina Kaif is collecting wedding presents from her fellow celeb\n",
      "permalink:   https://abbtakk.tv/en/salman-khan-gifts-range-rover-worth-rs30mn-to-katrina-kaif/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.tv/en/wp-content/uploads/2018/03/salman-khan-1.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       Trailer of ‘American Siege’ Released\n",
      "description: LOS ANGELES: We’re getting a first look at the trailer for the new Bruce Willis \n",
      "permalink:   https://abbtakk.tv/en/trailer-of-american-siege-released/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/17002042/american-siege-trailer-1-2022-mo.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       Animated Movie ‘The Bad Guys’ Trailer Released\n",
      "description: The Big Bad Wolf is coming soon to do more than blow a few houses down. Dreamwor\n",
      "permalink:   https://abbtakk.tv/en/animated-movie-the-bad-guys-trailer-released/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/15204142/The-Bad-Guys.png\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       India’s Harnaaz Sandhu Wins Miss Universe Contest\n",
      "description: Harnaaz Sandhu of India has been crowned the 70th Miss Universe, topping a field\n",
      "permalink:   https://abbtakk.tv/en/indias-harnaaz-sandhu-wins-miss-universe-contest/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/13204252/916444_8425298_Screenshot_113_updates.png)\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       Trailer of Operation Fortune Released\n",
      "description: Jason Statham is back doing what he does best—cracking heads while cracking wise\n",
      "permalink:   https://abbtakk.tv/en/trailer-of-operation-fortune-released/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/11134326/maxresdefault.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       Fawad Chaudhry Along With British HC Jointly Hosts Red Carpet Premiere of James Bond: No Time To Die\n",
      "description: ISLAMABAD: The premiere of one of the biggest movies since the pandemic, James B\n",
      "permalink:   https://abbtakk.tv/en/fawad-chaudhry-along-with-british-hc-jointly-hosts-red-carpet-premiere-of-james-bond-no-time-to-die/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/10212744/s300__D9A4730.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       Famous Film Actress Babra Sharif Celebrating Her 67th Birthday Today\n",
      "description: Lahore: Famous Pakistani Film Actress Babra Sharif is celebrating her 67th birth\n",
      "permalink:   https://abbtakk.tv/en/famous-film-actress-babra-sharif-celebrating-her-67th-birthday-today/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/10150743/Babra-Sharif.png\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       Riots Continue in Kazakhstan\n",
      "description: ALMATY: Bursts of gunfire echoed through the streets of Kazakhstan s largest cit\n",
      "permalink:   https://abbtakk.tv/en/riots-continue-in-kazakhstan/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06203108/Kazakhstan.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       Biden Condemns Trump as a Threat to Democracy\n",
      "description: President Joe Biden on Thursday marked the first anniversary of the January 6 in\n",
      "permalink:   https://abbtakk.tv/en/biden-condemns-trump-as-a-threat-to-democracy/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/07003245/donald-.jpeg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       Dozens of Policemen Dead in Clashes in Kazakhstan’s Almaty\n",
      "description: Dozens of protesters and at least a dozen security officers died in clashes in K\n",
      "permalink:   https://abbtakk.tv/en/dozens-of-policemen-dead-in-clashes-in-kazakhstans-almaty/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06203108/Kazakhstan.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       Eight Children Killed in US Apartment Fire\n",
      "description: WASHINGTON: Twelve people, including eight children, were killed early on Wednes\n",
      "permalink:   https://abbtakk.tv/en/eight-children-killed-in-us-apartment-fire/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06202433/us-fire.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          25\n",
      "title:       India PM Narendra Modi Trapped In Farmers Protest\n",
      "description: NEW DELHI: Indian Prime Minister Narendra Modi was trapped on a flyover for 20 m\n",
      "permalink:   https://abbtakk.tv/en/india-pm-narendra-modi-trapped-in-farmers-protest/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06001452/modi.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          26\n",
      "title:       Indian Forces Martyr Three Kashmiris in Occupied Kashmir\n",
      "description: SRINAGAR: In Indian illegally occupied Jammu and Kashmir, Indian troops in their\n",
      "permalink:   https://abbtakk.tv/en/indian-forces-martyr-three-kashmiris-in-occupied-kashmir/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.tv/en/wp-content/uploads/2021/07/Kashmir-1.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          27\n",
      "title:       Sudan’s PM Resigns\n",
      "description: KHARTOUM: Sudan’s civilian Prime Minister Abdalla Hamdok resigned Sunday, more t\n",
      "permalink:   https://abbtakk.tv/en/sudans-pm-resigns/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/03113519/635270_46481013.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          28\n",
      "title:       Stampede at Hindu Shrine Kills at Least 12 People in IoK\n",
      "description: SRINAGAR: At least 12 people have died and 13 injured in a stampede at a religio\n",
      "permalink:   https://abbtakk.tv/en/stampede-at-hindu-shrine-kills-at-least-12-people-in-iok/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/01125722/3447.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          29\n",
      "title:       Pakistani Youth Go To Bottom of Sea To Celebrate New Year\n",
      "description: RIYADH: In Saudi Arabia, Pakistani youth went to the bottom of the sea to celebr\n",
      "permalink:   https://abbtakk.tv/en/pakistani-youth-go-to-bottom-of-sea-to-celebrate-new-year/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/01000300/SA.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          30\n",
      "title:       New Zealand Welcomes New Year With Exquisite Light Show\n",
      "description: AUCKLAND: New Zealand on Friday welcomed the New Year 2022 with an exquisite lig\n",
      "permalink:   https://abbtakk.tv/en/new-zealand-welcomes-new-year-with-exquisite-light-show/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/31185738/auckland-new-year-1640950416.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          31\n",
      "title:       UN Chief Calls For Making 2022 As Year of Recovery\n",
      "description: NEW YORK: UN Secretary-General Antonio Guterres has urged the international comm\n",
      "permalink:   https://abbtakk.tv/en/un-chief-calls-for-making-2022-as-year-of-recovery/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/30212635/3082300751640843225.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          32\n",
      "title:       Travel Restrictions Covid-19: US President Revokes Southern Africa Travel Restrictions\n",
      "description: Washington: U.S. President Joe Biden has announced he will lift travel restricti\n",
      "permalink:   https://abbtakk.tv/en/travel-restrictions-covid-19-us-president-revokes-southern-africa-travel-restrictions/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/29185534/US-President.png\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          33\n",
      "title:       Saudi Arabia Once Again Made Wearing Of Mask Mandatory\n",
      "description: Riyadh: Saudi Arabia said that all citizens must wear masks at all times and mai\n",
      "permalink:   https://abbtakk.tv/en/__trashed-4/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/29193649/SAMM.png\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          34\n",
      "title:       Israeli Jets Attack Syrian Port\n",
      "description: An Israeli airstrike hit Syria’s Latakia port before dawn on Tuesday, sparking a\n",
      "permalink:   https://abbtakk.tv/en/israeli-jets-attack-syrian-port/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/29003702/132.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          35\n",
      "title:       Sputnik-V Vaccine More Effective Against Omicron\n",
      "description: MOSCOW: Russia’s Sputnik-V vaccine and its one-shot version Sputnik Light booste\n",
      "permalink:   https://abbtakk.tv/en/sputnik-v-vaccine-more-effective-against-omicron/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/29003210/1-545.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          36\n",
      "title:       US, Russia Set For Security Talks On January 10 Amid Ukraine Tensions\n",
      "description: WEB DESK: US and Russian officials have said that they will hold security talks \n",
      "permalink:   https://abbtakk.tv/en/us-russia-set-for-security-talks-on-january-10-amid-ukraine-tensions/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.tv/en/wp-content/uploads/2018/08/russia1.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          37\n",
      "title:       Iran Says Vienna Nuclear Talks to Restart Around Joint Document\n",
      "description: TEHRAN: The eighth round of Iran nuclear talks are starting on Monday in Vienna,\n",
      "permalink:   https://abbtakk.tv/en/iran-says-vienna-nuclear-talks-to-restart-around-joint-document/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2020/03/04003545/iran_atom_620x330.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          38\n",
      "title:       Indian Troops Martyr One More Kashmiri Youth In IIOJK\n",
      "description: Srinagar: The Indian forces martyred one more Kashmiri youth in south Kashmir’s \n",
      "permalink:   https://abbtakk.tv/en/indian-troops-martyr-one-more-kashmiri-youth-in-iiojk/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/03/11004455/kashmir_620x330-1.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          39\n",
      "title:       Christians In India Being Forced to Convert To Hinduism: New York Times\n",
      "description: NEW YORK: Terrifying revelations of atrocities against the Christian community i\n",
      "permalink:   https://abbtakk.tv/en/christians-in-india-being-forced-to-convert-to-hinduism-new-york-times/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/25185502/hindus.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          40\n",
      "title:       France Recommends Covid Booster Jab 3 Months After Initial Vaccines\n",
      "description: PARIS: France on Friday recommended that adults receive a Covid-19 booster vacci\n",
      "permalink:   https://abbtakk.tv/en/france-recommends-covid-booster-jab-3-months-after-initial-vaccines/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/25130915/634212_35137924.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          41\n",
      "title:       Pakistan Ready To Host SAARC Session: FO\n",
      "description: ISLAMABAD: Foreign Office spokesperson Asim Iftikhar Ahmed has categorically rej\n",
      "permalink:   https://abbtakk.tv/en/pakistan-ready-to-host-saarc-session-fo/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.tv/en/wp-content/uploads/2021/07/ForeignMinister.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          42\n",
      "title:       Tareen Attends Saad Rafique’s Daughter Marriage Ceremony\n",
      "description: LAHORE: Pakistan Tehreek-e-Insaf (PTI) leader Jahangir Tareen has attended the w\n",
      "permalink:   https://abbtakk.tv/en/tareen-attends-saad-rafiques-daughter-marriage-ceremony/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/05/03145614/Jahangir-Tareen.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          43\n",
      "title:       Hamid Yar Hiraj Sworn In As Provincial Minister of Punjab\n",
      "description: LAHORE: Another addition in Punjab cabinet after assembly member Sardar Hamid Ya\n",
      "permalink:   https://abbtakk.tv/en/hamid-yar-hiraj-sworn-in-as-provincial-minister-of-punjab/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/07155730/punjab.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          44\n",
      "title:       CM Seeks Report About Damages Occured in Balochistan Due to Rains\n",
      "description: ISLAMABAD: Prime Minister Imran Khan has sought report from Chief Minister Baloc\n",
      "permalink:   https://abbtakk.tv/en/cm-seeks-report-about-damages-occured-in-balochistan-due-to-rains/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.tv/en/wp-content/uploads/2021/09/Imran-Khan.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          45\n",
      "title:       IHC Defers Indictment of Rana Shamim, Others\n",
      "description: ISLAMABAD: The Islamabad High Court (IHC) on Friday deferred indictment in the c\n",
      "permalink:   https://abbtakk.tv/en/ihc-defers-indictment-of-rana-shamim-others/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/11/30145018/rana-shamim.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          46\n",
      "title:       IHC Orders To Raze Naval Sailing Club\n",
      "description: ISLAMABAD: The Islamabad High Court (IHC) on Friday ordered to raze Naval Sailin\n",
      "permalink:   https://abbtakk.tv/en/ihc-orders-to-raze-naval-sailing-club/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/07154211/naval.jpeg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          47\n",
      "title:       Gwadar Declared Calamity-Hit After Heavy Rain\n",
      "description: QUETTA: Many areas of northern Balochistan have been cut off from Quetta as vehi\n",
      "permalink:   https://abbtakk.tv/en/gwadar-declared-calamity-hit-after-heavy-rain/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/07122822/20220107_122754.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          48\n",
      "title:       COVID-19 Cases on Rise in Pakistan\n",
      "description: ISLAMABAD:Pakistan has reported 6 deaths in the last 24 hours by novel coronavir\n",
      "permalink:   https://abbtakk.tv/en/covid-19-cases-on-rise-in-pakistan/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/29142827/Coronavirus-2.png\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          49\n",
      "title:       Pakistan To Host OIC Meeting in March: FM Qureshi\n",
      "description: ISLAMABAD: Pakistan is gearing up to host the next Organisation of Islamic Coope\n",
      "permalink:   https://abbtakk.tv/en/pakistan-to-host-oic-meeting-in-march-fm-qureshi/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06235133/391940_8525972_updates.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          50\n",
      "title:       Govt To Complete Its Tenure: PM Imran Khan\n",
      "description: ISLAMABAD: Prime Minister Imran Khan on Thursday claimed that his government wil\n",
      "permalink:   https://abbtakk.tv/en/govt-to-complete-its-tenure-pm-imran-khan/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.tv/en/wp-content/uploads/2021/09/Imran-Khan.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          51\n",
      "title:       IMF Programme To Bring Stability in Pakistan: Fawad\n",
      "description: ISLAMABAD: Minister for Information and Broadcasting Chaudhry Fawad Hussain has \n",
      "permalink:   https://abbtakk.tv/en/imf-programme-to-bring-stability-in-pakistan-fawad/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/28225013/fawad.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          52\n",
      "title:       SC Accepts Apology of NAB Officials in Contempt Case\n",
      "description: ISLAMABAD: The Supreme Court (SC) on Thursday accepted the unconditional apology\n",
      "permalink:   https://abbtakk.tv/en/sc-accepts-apology-of-nab-officials-in-contempt-case/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/16152636/SC-1.png\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          53\n",
      "title:       JCP Okays Appointment of Justice Ayesha Malik To SC\n",
      "description: ISLAMABAD: The Judicial Commission of Pakistan (JCP) on Thursday recommended the\n",
      "permalink:   https://abbtakk.tv/en/jcp-okays-appointment-of-justice-ayesha-malik-to-sc/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06220803/malik-768x422-1.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          54\n",
      "title:       President Alvi Tests Positive for COVID-19\n",
      "description: President Dr Arif Alvi on Thursday again tested positive for coronavirus.The pre\n",
      "permalink:   https://abbtakk.tv/en/president-alvi-tests-positive-for-covid-19/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.tv/en/wp-content/uploads/2021/08/Arif-Alvi-1.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          55\n",
      "title:       PM Used Religion To Cover His Wrongdoings: Maryam Nawaz\n",
      "description: LAHORE: Pakistan Muslim League-Nawaz Vice President Maryam Nawaz on Thursday all\n",
      "permalink:   https://abbtakk.tv/en/pm-used-religion-to-cover-his-wrongdoings-maryam-nawaz/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06201852/Maryam-Nawaz-768x422-1.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          56\n",
      "title:       Deputy Chief of Turkish General Staff Calls On COAS Bajwa\n",
      "description: Islamabad: Deputy Chief of Turkish General Staff General Selcuk Bayraktaroglu ca\n",
      "permalink:   https://abbtakk.tv/en/deputy-chief-of-turkish-general-staff-calls-on-coas-bajwa/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06194526/10186203021641477377.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          57\n",
      "title:       PPP Announces Long March Against Govt From Feb 27\n",
      "description: Lahore: Chairman of Pakistan People’s Party (PPP) Bilawal Bhutto Zardari has ann\n",
      "permalink:   https://abbtakk.tv/en/ppp-announces-long-march-against-govt-from-feb-27/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06193552/PPP.png\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          58\n",
      "title:       Urban Development Projects Key Priorities Of Govt: PM\n",
      "description: Islamabad: Prime Minister Imran Khan said urban development through modern real \n",
      "permalink:   https://abbtakk.tv/en/urban-development-projects-key-priorities-of-govt-pm/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06185830/1641476107.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          59\n",
      "title:       Situation Of Sindh Has Become Out Of Control: Mustafa Kamal\n",
      "description: Islamabad: Pak Sarzameen Party ( PSP) chief Syed Mustafa Kamal has said that the\n",
      "permalink:   https://abbtakk.tv/en/situation-of-sindh-has-become-out-of-control-mustafa-kamal/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06184654/SMK.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          60\n",
      "title:       Omicron Virus Confirmed In Dr Yasmin Rashid\n",
      "description: Lahore: Punjab Minister for Health Dr Yasmin Rashid has diagnosed with omicron v\n",
      "permalink:   https://abbtakk.tv/en/omicron-virus-confirmed-in-dr-yasmin-rashid/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/05/30170319/yasmin-Rashid.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          61\n",
      "title:       SA Beat India in Second Test\n",
      "description: JOHANNESBURG: South Africa captain Dean Elgar hit a superb 96 not out to guide S\n",
      "permalink:   https://abbtakk.tv/en/sa-beat-india-in-second-test/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/07160737/635819_59189173.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          62\n",
      "title:       Mohammad Rizwan Wins Most Valuable Cricketer of the Year Award\n",
      "description: LAHORE: Pakistan Cricket Board (PCB) has announced the winners of PCB Awards 202\n",
      "permalink:   https://abbtakk.tv/en/mohammad-rizwan-wins-most-valuable-cricketer-of-the-year-award/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/07002533/PCB-Awards-696x456-1.png\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          63\n",
      "title:       PCB Awards 2021 Taking Place Today\n",
      "description: LAHORE: The Pakistan Cricket Board (PCB) Awards 2021 — aimed at recognising, app\n",
      "permalink:   https://abbtakk.tv/en/pcb-awards-2021-taking-place-today/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2019/08/22165301/PCB-3-1.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          64\n",
      "title:       PCB Readies COVID-19 SOPs For PSL-7\n",
      "description: LAHORE: In a bid to ensure that all the matches of the upcoming season 7 of the \n",
      "permalink:   https://abbtakk.tv/en/pcb-readies-covid-19-sops-for-psl-7/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/06001934/PSL.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          65\n",
      "title:       Hafeez Body Language Doesn’t Match With His Retirement Decision: Shahid Afridi\n",
      "description: Former skipper Shahid Afridi has said despite veteran cricketer Mohammad Hafeez’\n",
      "permalink:   https://abbtakk.tv/en/hafeez-body-language-doesnt-match-with-his-retirement-decision-shahid-afridi/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/05005935/dc-Cover-lt4q7fvo0ng83s6rlukl222252-20160508183620.Medi_.jpeg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          66\n",
      "title:       Babar, Rizwan, Shaheen, Fawad Selected For PCB Awards 2021\n",
      "description: Lahore: Star cricketers Babar Azam, Mohammad Rizwan, Shaheen Afridi, Fawad Alam \n",
      "permalink:   https://abbtakk.tv/en/babar-rizwan-shaheen-fawad-selected-for-pcb-awards-2021/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/04195813/PCB.png\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          67\n",
      "title:       Hafeez Announces Retirement From International Cricket\n",
      "description: LAHORE: Former captain Mohammad Hafeez has announced his retirement from interna\n",
      "permalink:   https://abbtakk.tv/en/hafeez-announces-retirement-from-international-cricket/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2020/06/16195935/Mohmmad-Hafeez.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          68\n",
      "title:       44th Edition Of Dakar Rally Commences In Saudi Arabia\n",
      "description: Riyadh: The 44th edition of the Dakar Rally got off to a great start in the high\n",
      "permalink:   https://abbtakk.tv/en/44th-edition-of-dakar-rally-commences-in-saudi-arabia/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/02175908/Dakar-Rally.png\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          69\n",
      "title:       Pakistan U-19 Team Leaves West Indies To Participate In Junior Cricket World Cup\n",
      "description: LAHORE: Pakistan Under-19 team has left for the West Indies to participate in th\n",
      "permalink:   https://abbtakk.tv/en/pakistan-u-19-team-leaves-to-participate-in-icc-junior-cricket-world-cup/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/02203532/U19.png\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          70\n",
      "title:       India Beat SL To Lift U-19 Asia Cup\n",
      "description: DUBAI: Year 2021 ended on the right note for the U-19 Indian cricket team as it \n",
      "permalink:   https://abbtakk.tv/en/india-beat-sl-to-lift-u-19-asia-cup/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2022/01/01172510/fh6jbi24_u19-indian-cricket-team_625x300_31_December_21.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          71\n",
      "title:       PCB Chairman Calls on PM Imran\n",
      "description: LAHORE: Prime Minister Imran Khan has accorded approval in principle to construc\n",
      "permalink:   https://abbtakk.tv/en/pcb-chairman-calls-on-pm-imran/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/31204734/de.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          72\n",
      "title:       Shaheen, Rizwan Nominated By ICC For Sir Garfield Sobers Trophy\n",
      "description: Dubai: The International Cricket Council (ICC) has issued a list of its nominees\n",
      "permalink:   https://abbtakk.tv/en/shaheen-rizwan-nominated-by-icc-for-sir-garfield-sobers-trophy/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/31160634/download-4.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          73\n",
      "title:       KP Lifts Quaid-e-Azam Trophy\n",
      "description: KARACHI: Khyber Pakhtunkhwa won the Quaid-e-Azam Trophy 2021-22, after defeating\n",
      "permalink:   https://abbtakk.tv/en/kp-lifts-quaid-e-azam-trophy/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/30213613/KPK-768x422-1.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          74\n",
      "title:       Brisbane Heat Signs Fakhar Zaman for Big Bash League\n",
      "description: Canberra: Big Bash League outfit, Brisbane Heat has signed explosive Pakistani b\n",
      "permalink:   https://abbtakk.tv/en/brisbane-heat-signs-fakhar-zaman-for-big-bash-league/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/30175130/Fakhar-Zaman-696x463-1.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          75\n",
      "title:       New Zealand’s Ross Taylor Announces Retirement From International Cricket\n",
      "description: WELLINGTON: New Zealand cricket icon Ross Taylor confirmed Thursday he is bowing\n",
      "permalink:   https://abbtakk.tv/en/new-zealands-ross-taylor-announces-retirement-from-international-cricket/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/30163835/Untitled.png\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          76\n",
      "title:       PSL 2022: NCOC Allows 100pc Crowd\n",
      "description: ISLAMABAD: Following the government s official go-ahead, the Pakistan Super Leag\n",
      "permalink:   https://abbtakk.tv/en/psl-2022-ncoc-allows-100pc-crowd/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/30003718/634812_74769193.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          77\n",
      "title:       PCB Decides To Appoint Five Coaches For National Team\n",
      "description: Lahore: The Pakistan Cricket Board (PCB) has decided to appoint five new coaches\n",
      "permalink:   https://abbtakk.tv/en/pcb-decides-to-appoint-five-coaches-for-national-team/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/04/11232130/PCB.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          78\n",
      "title:       ICC Announces Nominees For Men’s Test Player Of The Year\n",
      "description: Dubai: The International Cricket Council (ICC) has announced the nominees for Me\n",
      "permalink:   https://abbtakk.tv/en/icc-announces-nominees-for-mens-test-player-of-the-year/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/06/02215311/ICC-1.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          79\n",
      "title:       Day Three of Quaid-e-Azam Trophy Final Washed Out\n",
      "description: KARACHI: The third day of the Quaid-e-Azam Trophy final between Khyber Pakhtunkh\n",
      "permalink:   https://abbtakk.tv/en/day-three-of-quaid-e-azam-trophy-final-washed-out/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.tv/en/wp-content/uploads/2018/03/National-Stadium-Karachi.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          80\n",
      "title:       Asia Cup U19: Pakistan Beat UAE To Reach Semi-Final\n",
      "description: DUBAI: Pakistan on Monday qualified for the semi-final after beating UAE by 21 r\n",
      "permalink:   https://abbtakk.tv/en/asia-cup-u19-pakistan-beat-uae-to-reach-semi-final/\n",
      "channel:     AbbTakk\n",
      "image:       https://abbtakk.s3.ap-southeast-1.amazonaws.com/en/wp-content/uploads/2021/12/27203444/1-3.jpg\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 80\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources22 = [\n",
    "                ['https://abbtakk.tv/en/category/entertainment/','Entertainment','AbbTakk'],\n",
    "                 ['https://abbtakk.tv/en/category/world/','International','AbbTakk'],\n",
    "                 ['https://abbtakk.tv/en/category/pakistan/','National-local','AbbTakk'],\n",
    "                 ['https://abbtakk.tv/en/category/sports/','Sports','AbbTakk'],\n",
    "                ]\n",
    "    if Abbtak == True:\n",
    "        print('Getting Abbtakk News...')\n",
    "        counter = 0\n",
    "        i = 1\n",
    "        permalinkArrayAbbtakNews = []\n",
    "        for source in sources22:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('article',class_='penci-imgtype-landscape'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = article.find('div', class_='entry-text').header.h2.a['href']\n",
    "                        permalinkArrayAbbtakNews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "                i = 1\n",
    "        detail_source = permalinkArrayAbbtakNews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for main in soup.find_all('main'):\n",
    "                try:\n",
    "                    detail_title = main.find('div',class_='entry-media').div.div.find('div',class_='penci-featured-col-1').div.div.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    high_resolution_image = main.find('div',class_='entry-media').div.div.find('div',class_='penci-featured-col-2').div.div['style'][22:-24]       \n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = main.find('div',class_='penci-entry-content entry-content').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }      \n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))   \n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting 92 News...\n",
      "total news found on listting page : 48\n",
      "\n",
      "No:          1\n",
      "title:       Imran Niazi should resign after being proved 'thief' and 'liar': Shehbaz Sharif\n",
      "description: ISLAMABAD (92 News) – Opposition Leader in the National Assembly Shehbaz Sharif \n",
      "permalink:   https://92newshd.tv/about/imran-niazi-should-resign-after-being-proved-thief-and-liar-shehbaz-sharif\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07125738_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       COAS Qamar Bajwa, UK High Commissioner Christian Turner discuss Afghanistan situation\n",
      "description: RAWALPINDI (92 News) – Christian Turner, British High Commissioner, called on Ch\n",
      "permalink:   https://92newshd.tv/about/coas-qamar-bajwa-uk-high-commissioner-christian-turner-discuss-afghanistan-situation\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07122311_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       Pak Army, Navy and FC troops continue relief operation in Balochistan's coastal belt\n",
      "description: QUETTA (92 News) – Pakistan Army, Navy and FC troops are busy in rescue and reli\n",
      "permalink:   https://92newshd.tv/about/pak-army-navy-and-fc-troops-continue-relief-operation-in-balochistan-s-coastal-belt\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07120149_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       Man throws three daughters into canal in Chunian\n",
      "description: CHAUNIAN (92 News) – A man shot his three teenage daughters after throwing them \n",
      "permalink:   https://92newshd.tv/about/man-shoots-three-daughters-after-throwing-them-into-canal-in-chunian\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07112535_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       Urea shortage impossible in Pakistan, claims Khusro Bakhtiar\n",
      "description: ISLAMABAD (92 News) – Federal Minister for Industries and Production Khusro Bakh\n",
      "permalink:   https://92newshd.tv/about/urea-shortage-impossible-in-pakistan-claims-khusro-bakhtiar\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07122621_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       IHC declares construction of Navy Club along Rawal Lake as illegal\n",
      "description: ISLAMABAD (92 News) – The Islamabad High Court (IHC) on Friday declared the cons\n",
      "permalink:   https://92newshd.tv/about/ihc-declares-construction-of-navy-club-along-rawal-lake-as-illegal\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07101205_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       Roof of Masjid Al Nabawi reopened to worshippers\n",
      "description: RIYADH (92 News) – The Saudi authorities reopened the roof of Masjid Al Nabawi t\n",
      "permalink:   https://92newshd.tv/about/roof-of-masjid-al-nabawi-reopened-to-worshippers\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07093045_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       IHC adjourns indictment of Rana Shamim, others till Jan 20 in contempt of court case\n",
      "description: ISLAMABAD (92 News) – The Islamabad High Court (IHC) on Friday adjourned the ind\n",
      "permalink:   https://92newshd.tv/about/ihc-adjourns-indictment-of-rana-shamim-others-till-jan-20-in-contempt-of-court-case\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07091326_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Schools reopen with SOPs across Punjab\n",
      "description: LAHORE (92 News) – All government and private schools across Punjab province hav\n",
      "permalink:   https://92newshd.tv/about/schools-reopen-with-sops-across-punjab\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07050540_25.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       Fawad Chaudhary says families rule in PPP, PML-N is about to end\n",
      "description: ISLAMABAD (92 News) – Federal Minister for Information Fawad Chaudhary has said \n",
      "permalink:   https://92newshd.tv/about/fawad-chaudhary-says-families-rule-in-ppp-pml-n-is-about-to-end\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/06161428_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       Bilawal Bhutto announces anti-govt movement from Lahore\n",
      "description: LAHORE (92 News) – PPP chairman Bilawal Bhutto has announced to launch an anti-g\n",
      "permalink:   https://92newshd.tv/about/bilawal-bhutto-announces-anti-govt-movement-from-lahore\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/06150412_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       Imran Khan should be punished for his corruption in foreign funding case: Maryam Nawaz\n",
      "description: ISLAMABAD (92 News) – PML-N vice-president Maryam Nawaz said that Prime Minister\n",
      "permalink:   https://92newshd.tv/about/imran-khan-received-illegal-funds-bribe-in-name-of-donation-maryam-nawaz\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/06135711_4.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       Djokovic waits in Australian detention as lawyers debate vaccination status\n",
      "description: MELBOURNE (Reuters) – World number one Novak Djokovic spent a second day in immi\n",
      "permalink:   https://92newshd.tv/about/djokovic-waits-in-australian-detention-as-lawyers-debate-vaccination-status\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07063106_25.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       Outcry as Australia bars Serbian tennis player Djokovic over vaccination status\n",
      "description: MELBOURNE (Reuters) – World number one tennis player Novak Djokovic was denied e\n",
      "permalink:   https://92newshd.tv/about/outcry-as-australia-bars-serbian-tennis-player-djokovic-over-vaccination-status\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/06051452_42.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       Bangladesh's Mominul hails 'unbelievable' Test win in New Zealand\n",
      "description: MOUNT MAUNGANUI (AFP) – An ecstatic Bangladesh captain Mominul Haque was lost fo\n",
      "permalink:   https://92newshd.tv/about/bangladesh-s-mominul-hails-unbelievable-test-win-in-new-zealand\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/05044739_42.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       Football manager Ralf Rangnick suffers first defeat as Wolves win at United\n",
      "description: MANCHESTER (Reuters) – Manchester United suffered their first defeat under inter\n",
      "permalink:   https://92newshd.tv/about/football-manager-ralf-rangnick-suffers-first-defeat-as-wolves-win-at-united\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/04071539_42.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       Depth of talent bodes well for women's game, says Australian tennis player Ashleigh Barty\n",
      "description: MELBOURNE (Reuters) - World number one Ashleigh Barty said 2021 had been a big y\n",
      "permalink:   https://92newshd.tv/about/depth-of-talent-bodes-well-for-women-s-game-says-australian-tennis-player-ashleigh-barty\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/03112827_42.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       Former Australia rugby league international Blake Ferguson arrests on drugs charges\n",
      "description: TOKYO (AFP) – Ferguson was \"arrested on suspicion of possessing illegal drugs,\" \n",
      "permalink:   https://92newshd.tv/about/former-australia-rugby-league-international-blake-ferguson-arrests-on-drugs-charges\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/02053513_42.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       Crawley prepared to do whatever it takes to play third Ashes test\n",
      "description: SYDNEY (Reuters) - England opener Zak Crawley said he would be prepared to endur\n",
      "permalink:   https://92newshd.tv/about/crawley-prepared-to-do-whatever-it-takes-to-play-third-ashes-test\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/02035932_25.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       British tennis player Emma Raducanu pulls out of Australian Open warm-up event\n",
      "description: MELBOURNE (Reuters) - US Open champion Emma Raducanu has withdrawn from this wee\n",
      "permalink:   https://92newshd.tv/about/british-tennis-player-emma-raducanu-pulls-out-of-australian-open-warm-up-event\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/01070327_42.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       Australia's Head ruled out of Sydney Ashes Test with COVID-19\n",
      "description: CANBERRA (Reuters) – Australia batsman Travis Head will miss the fourth Ashes Te\n",
      "permalink:   https://92newshd.tv/about/australia-s-head-ruled-out-of-sydney-ashes-test-with-covid-19\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/31103513_25.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       Manchester United up to sixth after 3-1 win over Burnley\n",
      "description: MANCHESTER (Reuters) – Manchester United moved up to sixth in the Premier League\n",
      "permalink:   https://92newshd.tv/about/manchester-united-up-to-sixth-after-3-1-win-over-burnley\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/31101609_25.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       Sri Lanka beat Pakistan to book ACC Under-19 Asia Cup final spot\n",
      "description: DUBAI (92 News) – Sri Lanka defeated Pakistan by 22 runs to book their place in \n",
      "permalink:   https://92newshd.tv/about/sri-lanka-beat-pakistan-to-book-acc-under-19-asia-cup-final-spot\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/30154847_4.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       Khyber Pakhtunkhwa beat Northern to win Quaid-e-Azam Trophy title\n",
      "description: KARACHI (92 News) – Khyber Pakhtunkhwa won the prestigious Quaid-e-Azam Trophy a\n",
      "permalink:   https://92newshd.tv/about/khyber-pakhtunkhwa-beat-northern-to-win-quaid-e-azam-trophy-title\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/29153653_4.webp\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          25\n",
      "title:       India's COVID-19 cases jump to 117,100 as Omicron spreads\n",
      "description: NEW DELHI (Reuters) – India's daily COVID-19 cases jumped to 117,100 on Friday, \n",
      "permalink:   https://92newshd.tv/about/india-s-covid-19-cases-jump-to-117-100-as-omicron-spreads\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07103740_4.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          26\n",
      "title:       Omicron may be less severe in young and old, but not 'mild' - WHO\n",
      "description: GENEVA (Reuters) – The more infectious Omicron variant of COVID-19 appears to pr\n",
      "permalink:   https://92newshd.tv/about/omicron-may-be-less-severe-in-young-and-old-but-not-mild-who\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/07042232_25.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          27\n",
      "title:       60 percent Omicron cases are being reported only in Karachi and Lahore: Asad Umar\n",
      "description: LAHORE (92 News) – Minister for Planning and Development Asad Umar has said that\n",
      "permalink:   https://92newshd.tv/about/60-percent-omicron-cases-are-being-reported-only-in-karachi-and-lahore-asad-umar\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/05102306_42.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          28\n",
      "title:       Tempers fraying in Australia as COVID-19 cases hit new records\n",
      "description: SYDNEY (Reuters) - Australia's daily COVID-19 cases hit a record high for the th\n",
      "permalink:   https://92newshd.tv/about/tempers-fraying-in-australia-as-covid-19-cases-hit-new-records\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/05040201_42.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          29\n",
      "title:       COVID-19 claims two lives, 630 new cases reported during last 24 hours\n",
      "description: ISLAMABAD (92 News) – As many as two people have lost their lives due to COVID-1\n",
      "permalink:   https://92newshd.tv/about/covid-19-claims-two-lives-630-new-cases-reported-during-last-24-hours\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/04054959_42.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          30\n",
      "title:       COVID-19 claims two lives, 708 new cases reported during last 24 hours\n",
      "description: ISLAMABAD (92 News) – As many as two people have lost their lives due to COVID-1\n",
      "permalink:   https://92newshd.tv/about/covid-19-claims-two-lives-708-new-cases-reported-during-last-24-hours\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/03050347_42.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          31\n",
      "title:       Australia to push ahead with reopening amid record COVID-19 cases\n",
      "description: SYDNEY (Reuters) - Australia's government said the milder impact of the Omicron \n",
      "permalink:   https://92newshd.tv/about/australia-to-push-ahead-with-reopening-amid-record-covid-19-cases\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/03041211_25.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          32\n",
      "title:       Asad Umar warns against spread of Omicron in country\n",
      "description: ISLAMABAD (92 News) – Planning Minister Asad Umar has warned against spread of a\n",
      "permalink:   https://92newshd.tv/about/asad-umar-warns-against-spread-of-omicron-in-country\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/02164002_25.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          33\n",
      "title:       Daily COVID cases in Saudi Arabia above 1,000, continue to climb in UAE\n",
      "description: ABU DHABI (Reuters) – Daily coronavirus cases in Saudi Arabia have climbed above\n",
      "permalink:   https://92newshd.tv/about/daily-covid-cases-in-saudi-arabia-above-1-000-continue-to-climb-in-uae\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/02115153_42.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          34\n",
      "title:       Health card will be for every rich and poor citizen, says Yasmin Rashid\n",
      "description: LAHORE (92 News) – Minister for Health Punjab Dr Yasmin Rashid has said that hea\n",
      "permalink:   https://92newshd.tv/about/heal-card-will-be-for-every-rich-and-poor-citizen-says-yasmin-rashid\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/02075631_42.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          35\n",
      "title:       COVID-19 claims eight more lives, 594 new cases reported during last 24\n",
      "description: ISLAMABAD (92 News) – As many as eight people have lost their lives due to COVID\n",
      "permalink:   https://92newshd.tv/about/covid-19-claims-eight-more-lives-594-new-cases-reported-during-last-24\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/02055843_42.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          36\n",
      "title:       New Australian COVID-19 cases dip, but hospitalisations rise\n",
      "description: SYDNEY (Reuters) – New Australian COVID-19 cases dipped on Sunday as testing slo\n",
      "permalink:   https://92newshd.tv/about/new-australian-covid-19-cases-dip-but-hospitalisations-rise\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/02033304_25.webp\n",
      "category:    Health\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          37\n",
      "title:       Music's Grammy Awards indefinitely postponed as Omicron spreads\n",
      "description: LOS ANGELES (Reuters) - This year's Grammy Awards ceremony honoring top performa\n",
      "permalink:   https://92newshd.tv/about/music-s-grammy-awards-indefinitely-postponed-as-omicron-spreads\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/06122444_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          38\n",
      "title:       American actress Betty White has a last laugh in biographic comic book\n",
      "description: PORTLAND (Reuters) – The life of comedic actress Betty White is being celebrated\n",
      "permalink:   https://92newshd.tv/about/american-actress-betty-white-has-a-last-laugh-in-biographic-comic-book\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/05090941_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          39\n",
      "title:       Catalog of late rocker David Bowie sold to Warner Music\n",
      "description: LONDON (Reuters) - Warner Music Group's publishing unit has bought late British \n",
      "permalink:   https://92newshd.tv/about/catalog-of-late-rocker-david-bowie-sold-to-warner-music\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2022/01/04051249_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          40\n",
      "title:       American actor Arnold Schwarzenegger and Maria Shriver are finally divorced\n",
      "description: LOS ANGELES (Reuters) – American actor Arnold Schwarzenegger and American journa\n",
      "permalink:   https://92newshd.tv/about/american-actor-arnold-schwarzenegger-and-maria-shriver-are-finally-divorced\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/30081755_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          41\n",
      "title:       'Spider-Man: No Way Home' becomes first pandemic-era movie to smash $1 billion milestone globally\n",
      "description: LOS ANGELES (Reuters) - \"Spider-Man: No Way Home\" unwrapped the best Christmas g\n",
      "permalink:   https://92newshd.tv/about/spider-man-no-way-home-becomes-first-pandemic-era-movie-to-smash-1-billion-milestone-globally\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/27114341_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          42\n",
      "title:       71st birth anniversary of comedian Moin Akhtar observed\n",
      "description: KARACHI (92 News) – The 71st birth anniversary of Pakistani entertainment icon M\n",
      "permalink:   https://92newshd.tv/about/71st-birth-anniversary-of-moin-akhtar-being-observed-today\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/24043315_25.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          43\n",
      "title:       Mahershala Ali moves into the spotlight in 'Swan Song'\n",
      "description: NEW YORK (Reuters) – Double Oscar winner Mahershala Ali has finally found the ri\n",
      "permalink:   https://92newshd.tv/about/mahershala-ali-moves-into-the-spotlight-in-swan-song\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/23103134_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          44\n",
      "title:       'A Journal for Jordan' brings a father's life lessons for son to screen\n",
      "description: NEW YORK (Reuters) – Oscar winning actor Denzel Washington returns to the direct\n",
      "permalink:   https://92newshd.tv/about/a-journal-for-jordan-brings-a-father-s-life-lessons-for-son-to-screen\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/22083831_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          45\n",
      "title:       Hollywood movie 'Titane' director Ducournau flies flag for genre films, female voices\n",
      "description: LONDON (Reuters) – Director and writer Julia Ducournau says France's decision to\n",
      "permalink:   https://92newshd.tv/about/hollywood-movie-titane-director-ducournau-flies-flag-for-genre-films-female-voices\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/20122221_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          46\n",
      "title:       British singers Adele and Ed Sheeran lead nominees for BRIT Awards\n",
      "description: LONDON (Reuters) - Adele and Ed Sheeran were announced as two of the top nominee\n",
      "permalink:   https://92newshd.tv/about/british-singers-adele-and-ed-sheeran-lead-nominees-for-brit-awards\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/19065120_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          47\n",
      "title:       After a long lockdown, life returns to normalcy in Malaysia\n",
      "description: KUALA LUMPUR (92 News) - After a long lockdown, life in Malaysia has begun to re\n",
      "permalink:   https://92newshd.tv/about/after-a-long-lockdown-life-returns-to-normalcy-in-malaysia\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/18115128_42.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          48\n",
      "title:       6th death anniversary of renowned dramatist Kamal Ahmed Rizvi observed\n",
      "description: KARACHI (92 News) – The sixth death anniversary of renowned dramatist & actor Ka\n",
      "permalink:   https://92newshd.tv/about/sixth-death-anniversary-of-renowned-dramatist-kamal-ahmed-rizvi-being-observed\n",
      "channel:     92News\n",
      "image:       https://92newshd.tv/wp-content/uploads/2021/12/17060429_25.webp\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 48\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources19 = [\n",
    "                 ['https://92newshd.tv/category/pakistan','National-local','92News'],\n",
    "                 ['https://92newshd.tv/category/sports','Sports','92News'],\n",
    "                 ['https://92newshd.tv/category/Health','Health','92News'],\n",
    "                 ['https://92newshd.tv/category/Entertainment','Entertainment','92News']\n",
    "                ]\n",
    "    if Ninetytwonews == True:\n",
    "        counter = 0\n",
    "        print('Getting 92 News...')\n",
    "        i = 1\n",
    "        permalinkArray92News = []\n",
    "        for source in sources19:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='sub-posts'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = 'https://92newshd.tv' + div.a['href']\n",
    "                        permalinkArray92News.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "                i = 1\n",
    "        detail_source = permalinkArray92News\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='post-details'):\n",
    "                try:\n",
    "                    detail_title = div.h1.text\n",
    "\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    high_resolution_image = div.img['data-src']      \n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='content_detail').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1 \n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter)) \n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources15 = [\n",
    "                 ['https://www.24newshd.tv/pakistan','Local','24NewsHD'],\n",
    "                 ['https://www.24newshd.tv/world','International','24NewsHD'],\n",
    "                 ['https://www.24newshd.tv/sports','Sports','24NewsHD'],\n",
    "                 ['https://www.24newshd.tv/entertainment','Entertainment','24NewsHD'],\n",
    "                 ['https://www.24newshd.tv/opinion','Blog','24NewsHD']\n",
    "                ]\n",
    "    if Twentyfournews == True:\n",
    "        counter = 0\n",
    "        print('Getting 24News Channel...')\n",
    "        i = 1\n",
    "        permalinkArray24News = []\n",
    "        for source in sources15:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('div',class_='category-post-block'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = article.div.div.a['href']\n",
    "                        permalinkArray24News.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                        print(permalink)\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                        print('exp')\n",
    "                    i = i + 1\n",
    "                i = 1\n",
    "        detail_source = permalinkArray24News\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='detail-page'):\n",
    "                try:\n",
    "                    detail_title = div.h1.text\n",
    "\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    high_resolution_image = div.img['data-src']\n",
    "\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='content_detail').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                language='English'\n",
    "                counter = counter + 1\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }\n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if False:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting GNN News...\n",
      "total news found on listting page : 40\n",
      "\n",
      "No:          1\n",
      "title:       Pakistan on course to appoint first female Supreme Court judge\n",
      "description: Islamabad: Pakistan's judicial commission confirmed the nomination of the first \n",
      "permalink:   https://gnnhd.tv/news/9044\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12522/conversions/889548_9644738_Judges-start-saying-‘no’-to-govt-plots_akhbar-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       COVID-19: Pakistan reports 1,293 infections, six deaths\n",
      "description: Pakistan reported 1,293 new coronavirus infections, a new daily record, with aut\n",
      "permalink:   https://gnnhd.tv/news/9035\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12499/conversions/thumbs_b_c_f5a496f42d7b1df2aadb3d76b1786f1f-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       November is still away, PM answers about another extension to Gen Bajwa\n",
      "description: When asked about giving another extension to Chief of Army Staff (COAS) General \n",
      "permalink:   https://gnnhd.tv/news/9034\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12494/conversions/Imrankhan-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       Fawad suggests PML-N, PPP to replace names with ‘Sharifia’ and ‘Zardaria’\n",
      "description: Islamabad: Minister for Information and Broadcasting Fawad Chaudhry on Thursday \n",
      "permalink:   https://gnnhd.tv/news/9033\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12491/conversions/fawad-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       Incumbent government taking roof from over the poor’s head: Bilawal\n",
      "description: Lahore: Pakistan Peoples’ Party (PPP) Chairman Bilawal Bhutto has said that the \n",
      "permalink:   https://gnnhd.tv/news/9031\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12487/conversions/Bilawal-bhutto-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Pakistan to host once again OIC session: FM Qureshi\n",
      "description: Islamabad: Pakistan will host once again Organisation of Islamic Cooperation (OI\n",
      "permalink:   https://gnnhd.tv/news/9030\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12484/conversions/FM-Qureshi-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       International community’s help to Afghanistan imperative for regional peace: COAS\n",
      "description: Rawalpindi: Chief of Army Staff (COAS) General Qamar Javed Bajwa has said that t\n",
      "permalink:   https://gnnhd.tv/news/9057\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12550/conversions/COAS-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       Pakistan to resolutely expose India’s anti-peace agenda: FO Spokesperson\n",
      "description: Islamabad: The Foreign Office Spokesperson Friday said Pakistan will continue to\n",
      "permalink:   https://gnnhd.tv/news/9055\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12548/conversions/832245_6903763_FO_akhbar-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Bad weather forces PIA to divert three Lahore-bound flights to Islamabad Airport\n",
      "description: Islamabad: Three domestic and international PIA flights were forced to land at I\n",
      "permalink:   https://gnnhd.tv/news/9050\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12538/conversions/PIA-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       Issuing health cards is historic step towards Madina-like welfare state: Farrukh Habib\n",
      "description: Islamabad: Minister of State for Information and Broadcasting Farrukh Habib said\n",
      "permalink:   https://gnnhd.tv/news/9048\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12530/conversions/Farruk-Habib-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       Seven killed, 40 injured in Khairpur Tamewali’s bus crash\n",
      "description: Bahawalpur: As many as seven people were killed and 40 were injured in a bus cra\n",
      "permalink:   https://gnnhd.tv/news/8916\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12184/conversions/Accident-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       Public, private hospitals on Naya Pakistan Sehat Card panel increased from 43 to 150: CM Buzdar\n",
      "description: Lahore: Punjab Chief Minister (CM) Usman Buzdar has said that number of public a\n",
      "permalink:   https://gnnhd.tv/news/8864\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12056/conversions/CM-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       Sindh decides to start new academic year from April 1, 2022\n",
      "description: Karachi: The Sindh Education Department has decided to start the new academic ye\n",
      "permalink:   https://gnnhd.tv/news/8863\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12054/conversions/sindh-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       Mobile phones, currency notes showered in Sialkot’s unique wedding ceremony\n",
      "description: Sialkot: The guests have been welcomed amid showers of mobile phones and currenc\n",
      "permalink:   https://gnnhd.tv/news/8857\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12038/conversions/unique-wedding-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       APP journalist Fawad Azam dies in Jaranwala road mishap\n",
      "description: Faisalabad: Senior journalist Fawad Azam Awan and his fellow villager died in a \n",
      "permalink:   https://gnnhd.tv/news/8832\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/11977/conversions/Fawad-Azam-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       159 Hindu pilgrims arrive in Pakistan, leave for Peshawar for religious rituals\n",
      "description: Lahore: As many 159 Hindu devotees arrived in Pakistan from India via Wagah bord\n",
      "permalink:   https://gnnhd.tv/news/8830\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/11966/conversions/Hindu-pilgrims-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       Lahore shivers as intermittent rain adds to winter chills\n",
      "description: Lahore: Rain in the provincial capital Lahore and its adjacent areas and other c\n",
      "permalink:   https://gnnhd.tv/news/9038\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12510/conversions/499488_8381864_lahore2_akhbar-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       Two soldiers martyred, as many terrorists killed in KP operations\n",
      "description: Tank/South Waziristan: Two soldiers were martyred and as many terrorists killed \n",
      "permalink:   https://gnnhd.tv/news/8992\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12392/conversions/Operation-1-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       SNGPL team unearths gas theft at CNG station; employees, vehicles attacked\n",
      "description: Kasur: A Sui Northern Gas Pipelines Limited employee suffered gunshot wound afte\n",
      "permalink:   https://gnnhd.tv/news/8987\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12382/conversions/SNGPL-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       Four labourers die of suffocation in Karachi's garments factory\n",
      "description: Karachi: Four people died of suffocation in a factory near Chaukhandi Stop in Ka\n",
      "permalink:   https://gnnhd.tv/news/8919\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12187/conversions/Factory-suffocation-1280x720.webp\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       Webb telescope fully deploys sunshield in mission milestone\n",
      "description: Washington: The James Webb Space Telescope fully deployed its tennis-court sized\n",
      "permalink:   https://gnnhd.tv/news/8976\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12350/conversions/635575_18041216-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       Apple becomes world's first $3 trillion company\n",
      "description: Apple Inc has become the first company with a $3 trillion stock market value, li\n",
      "permalink:   https://gnnhd.tv/news/8924\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12201/conversions/8192-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       Tesla stock continues to rise following record vehicle deliveries in 2021\n",
      "description: Shares of Tesla continued to jump on Monday, a day after the electric vehicle co\n",
      "permalink:   https://gnnhd.tv/news/8907\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12169/conversions/Tesla-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       Beware of this dangerous message on WhatsApp! Simple negligence could empty your bank accounts\n",
      "description: The latest WhatsApp scam called Rediroff.com can be used to rob users of confide\n",
      "permalink:   https://gnnhd.tv/news/8898\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12142/conversions/Whatsapp-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          25\n",
      "title:       Tesla delivers almost 1 million cars globally\n",
      "description: Washington: Tesla said it delivered nearly one million vehicles in 2021, almost \n",
      "permalink:   https://gnnhd.tv/news/8885\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12106/conversions/2995146-1308101891-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          26\n",
      "title:       Twitter permanently bans US Representative Marjorie Taylor Greene\n",
      "description: Washington: Twitter on Sunday said it permanently banned the personal account of\n",
      "permalink:   https://gnnhd.tv/news/8872\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12076/conversions/Twitter-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          27\n",
      "title:       Apple sued over cosy search partnership with Google\n",
      "description: Google’s deal with Apple, which ensures that it remains the default search engin\n",
      "permalink:   https://gnnhd.tv/news/9016\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12443/conversions/fjgyjktu-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          28\n",
      "title:       France fines Google, Facebook $237 million over ‘cookies’\n",
      "description: Paris: French regulators have hit Google and Facebook with 210 million euros ($2\n",
      "permalink:   https://gnnhd.tv/news/9015\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12441/conversions/FB-Google-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          29\n",
      "title:       Italian mafia fugitive caught after two decades, thanks to Google Maps\n",
      "description: Rome: Italian police has caught a top mafia fugitive who had been on the run for\n",
      "permalink:   https://gnnhd.tv/news/9003\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12409/conversions/google-maps-ios-icon-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          30\n",
      "title:       Facebook removes Polish far-right political party account\n",
      "description: Warsaw: Facebook on Wednesday suspended the account of the Polish far-right part\n",
      "permalink:   https://gnnhd.tv/news/8990\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12386/conversions/facebook-1280x720.webp\n",
      "category:    IT-science\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          31\n",
      "title:       Gold price increases in Pakistan\n",
      "description: Karachi: The gold price on Wednesday increased by Rs100 per tola to Rs126,250 in\n",
      "permalink:   https://gnnhd.tv/news/8996\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12396/conversions/GOld-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          32\n",
      "title:       Apple becomes first company to hit $3 trillion market value, then slips\n",
      "description: San Francisco: Apple Inc on Monday became the first company to hit a $3 trillion\n",
      "permalink:   https://gnnhd.tv/news/8962\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12311/conversions/202-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          33\n",
      "title:       Pakistani rupee continues to slide against US dollar\n",
      "description: Karachi: The Pakistani rupee continued to lose ground and fell to Rs176.75 again\n",
      "permalink:   https://gnnhd.tv/news/8948\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12276/conversions/Dollar-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          34\n",
      "title:       Toyota overthrows GM to become US top-selling automaker in 2021\n",
      "description: Toyota Motor has dethroned General Motors as America’s top-selling automaker in \n",
      "permalink:   https://gnnhd.tv/news/8947\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12274/conversions/Toyota-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          35\n",
      "title:       Gold price declines in Pakistan\n",
      "description: Karachi: Gold prices in Pakistan recorded a decline of Rs150 per tola and Rs129 \n",
      "permalink:   https://gnnhd.tv/news/8944\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12267/conversions/GOld-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          36\n",
      "title:       Apple becomes first American company to hit $3 trillion market cap\n",
      "description: Apple briefly hit a market cap of $3 trillion during intraday trading on Monday,\n",
      "permalink:   https://gnnhd.tv/news/8917\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12185/conversions/Apple-1-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          37\n",
      "title:       Asian markets mixed as attention turns to US jobs\n",
      "description: Hong Kong: Asian equities were mixed Friday as investors battled to instigate a \n",
      "permalink:   https://gnnhd.tv/news/9039\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12511/conversions/635847_86445175-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          38\n",
      "title:       Gold price decreases in Pakistan\n",
      "description: Karachi: The gold price on Thursday decreased by Rs100 per tola to Rs126,150 in \n",
      "permalink:   https://gnnhd.tv/news/9026\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12467/conversions/GOld-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          39\n",
      "title:       US dollar continues to rise against Pakistani rupee\n",
      "description: Karachi: The Pakistani rupee continued to lose ground and fell to Rs176.92 again\n",
      "permalink:   https://gnnhd.tv/news/9023\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12456/conversions/Dollar-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          40\n",
      "title:       Iranian airline launches direct flight from Mashad to Karachi\n",
      "description: Karachi: Iranian National Airline 'Iran Air' has launched its direct flight from\n",
      "permalink:   https://gnnhd.tv/news/9009\n",
      "channel:     GNN\n",
      "image:       https://gnnhd.tv/media/12427/conversions/169346274-1280x720.webp\n",
      "category:    Business\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 40\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources11 = [\n",
    "                 ['https://gnnhd.tv/category/pakistan','National-local','GNN'],\n",
    "                 ['https://gnnhd.tv/category/regional','National-local','GNN'],\n",
    "                 ['https://gnnhd.tv/category/technology','IT-science','GNN'],\n",
    "                 ['https://gnnhd.tv/category/business','Business','GNN']\n",
    "                ]\n",
    "    if Gnnnews == True:\n",
    "        counter = 0\n",
    "        i = 1\n",
    "        print('Getting GNN News...')\n",
    "        permalinkArrayGNNnews = []\n",
    "        for source in sources11:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for a in soup.find('div',class_='mvp-feat5-small-sub').find_all('a'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = 'https://gnnhd.tv' + a['href']\n",
    "                        permalinkArrayGNNnews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "            for a in soup.find('div',class_='mvp-feat5-mid-sub-wrap').find_all('a'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = 'https://gnnhd.tv' + a['href']\n",
    "                        permalinkArrayGNNnews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayGNNnews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',{'id':'mvp-post-main'}):\n",
    "                try:\n",
    "                    detail_title = div.header.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    high_resolution_image = 'https://gnnhd.tv'+div.find('div',{'id':'mvp-post-feat-img'}).img['src'] \n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',{'id':'mvp-content-main'}).find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources9 = [\n",
    "                 ['https://en.dailypakistan.com.pk/pakistan','National-local','DailyPakistan'],\n",
    "                 ['https://en.dailypakistan.com.pk/sports','Sports','DailyPakistan'],\n",
    "                 ['https://en.dailypakistan.com.pk/world','International','DailyPakistan'],\n",
    "                 ['https://en.dailypakistan.com.pk/opinion','Blog','DailyPakistan'],\n",
    "                 ['https://en.dailypakistan.com.pk/national','National-local','DailyPakistan'],\n",
    "                 ['https://en.dailypakistan.com.pk/business','Business','DailyPakistan'],\n",
    "                ]\n",
    "    if Dailypakistan == True:\n",
    "        print('Getting Daily Pakistan News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayDPnews = []\n",
    "        for source in sources9:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='col-sm-6'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink =  div.div.div.find('div',class_='tt-news-img').a['href']\n",
    "                        permalinkArrayDPnews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayDPnews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',{'id':'mvp-post-main'}):\n",
    "                try:\n",
    "                    detail_title = div.header.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    high_resolution_image = 'https://gnnhd.tv'+div.find('div',{'id':'mvp-post-feat-img'}).img['src'] \n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',{'id':'mvp-content-main'}).find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }    \n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if False:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Newsone...\n",
      "exception in Newsone\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources18 = [\n",
    "                 ['https://www.newsone.tv/pakistan-news','National-local','Newsone'],\n",
    "                 ['https://www.newsone.tv/sports','Sports','Newsone'],\n",
    "                 ['https://www.newsone.tv/world','International','Newsone'],\n",
    "                 ['https://www.newsone.tv/blogs','Blog','Newsone'],\n",
    "                 ['https://www.newsone.tv/entertainment','Entertainment','Newsone'],\n",
    "                 ['https://www.newsone.tv/technology','IT-Science','Newsone'],\n",
    "                 ['https://www.newsone.tv/lifestyle','Health','Newsone'],\n",
    "                ]\n",
    "    if True == True:\n",
    "        print('Getting Newsone...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayNewsOne = []\n",
    "        for source in sources18:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('div',class_='eq-blocks'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink =  article.div.div.h2.a['href']\n",
    "                        permalinkArrayNewsOne.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1   \n",
    "        detail_source = permalinkArrayNewsOne\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='single-page-tem'):\n",
    "                try:\n",
    "                    detail_title = div.section.div.div.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    high_resolution_image = div.section.div.find('div', class_='single-post').find('div', class_='post-content').div.img['src'] \n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.section.div.find('div', class_='single-post').find('div', class_='post-content').article\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title.strip(),\n",
    "                            'description': detail_description.strip(),\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description.strip()[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Urdupoint News...\n",
      "total news found on listting page : 24\n",
      "\n",
      "No:          1\n",
      "title:       پاکستان تحریک انصاف کی تنظیم سازی ، شاہ محمود قریشی کو اہم عہدہ مل گیا\n",
      "description:  پاکستان تحریک انصاف  اسلام آباد (اُردو پوائنٹ تازہ ترین اخبار۔ 07 جنوری 2022ء) \n",
      "permalink:   https://www.urdupoint.com/pakistan/news/islamabad/important-news/live-news-3011225.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       مسلم لیگ ن سے پرانی دوستیاں ہیں جو ایسے ہی چلیں گی ، جہانگیر ترین\n",
      "description:  پاکستان تحریک انصاف  لاہور ( اُردو پوائنٹ اخبارتازہ ترین - 07 جنوری 2022ء )  پا\n",
      "permalink:   https://www.urdupoint.com/pakistan/news/lahore/important-news/live-news-3011223.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       ایف بی آر نے مکمل آمدن اور زرعی آمدن کو بھی ٹیکس ڈائریکٹری میں شامل کر لیا\n",
      "description:  پارلیمنٹ  کراچی(اُردو پوائنٹ اخبارتازہ ترین - این این آئی۔ 07 جنوری2022ء) ایف ب\n",
      "permalink:   https://www.urdupoint.com/pakistan/news/karachi/national-news/live-news-3011222.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       نوعمر لڑکی کے اندھے قتل کا معمہ حل، سگا باپ ہی قاتل نکلا\n",
      "description:  پولیس  کراچی(اُردو پوائنٹ اخبارتازہ ترین - این این آئی۔ 07 جنوری2022ء) شارع فیص\n",
      "permalink:   https://www.urdupoint.com/pakistan/news/karachi/national-news/live-news-3011221.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       روشن ڈیجیٹل اکائونٹ نے مزید 3 سنگ میل حاصل کر لیے، صرف 16 مہینوں میں 3 لاکھ سے زائد اکائونٹس کھولے گئے، اسٹیٹ بینک\n",
      "description:  پاکستان  کراچی(اُردو پوائنٹ اخبارتازہ ترین - این این آئی۔ 07 جنوری2022ء) اسٹیٹ \n",
      "permalink:   https://www.urdupoint.com/pakistan/news/karachi/national-news/live-news-3011220.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       گیس پریشر کم،شہرقائدکے شہری لکڑیاں جلانے پرمجبور\n",
      "description:  گیس  کراچی(اُردو پوائنٹ اخبارتازہ ترین - این این آئی۔ 07 جنوری2022ء)  گیس پریشر\n",
      "permalink:   https://www.urdupoint.com/pakistan/news/karachi/national-news/live-news-3011219.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       اعصام الحق نے اپنے ساتھی کے ساتھ اے ٹی پی میلبورن ٹینس ٹورنامنٹ کے مینز ڈبلز مقابلوں کے سیمی فائنل کیلئے کوالیفائی کرلیا\n",
      "description:  ٹینس  میلبور ن (اُردو پوائنٹ اخبارتازہ ترین - این این آئی۔ 07 جنوری2022ء) پاکست\n",
      "permalink:   https://www.urdupoint.com/sports/detail-news/live-news-3011173.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2020/420x350/pic_7763f_1583148690.jpg._2\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       نوواک جوکووچ کی آسٹریلیا سے بیدخلی پر’میمز‘ کا طوفان آگیا\n",
      "description:  طوفان  میلبور ن (اُردو پوائنٹ اخبارتازہ ترین - این این آئی۔ 07 جنوری2022ء) نووا\n",
      "permalink:   https://www.urdupoint.com/sports/detail-news/live-news-3011169.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_59aa6_1641542177.jpg._2\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       نیوزی لینڈ اور بنگلہ دیش کے درمیان دوسرا اور آخری ٹیسٹ میچ 9 جنوری سے شروع ہوگا\n",
      "description:  نیوزی لینڈ  سنچورین(اُردو پوائنٹ اخبارتازہ ترین - این این آئی۔ 07 جنوری2022ء)  \n",
      "permalink:   https://www.urdupoint.com/sports/detail-news/live-news-3011168.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_b4399_1641379133.jpg._2\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       آئی سی سی کا سلو اوور ریٹ کی سزا گرائونڈ پر ہی دینے کا اعلان\n",
      "description:  کرکٹ  لاہور (اردو پوائنٹ تازہ ترین اخبار ۔ 7 جنوری 2022ء ) انٹرنیشنل کرکٹ کونسل\n",
      "permalink:   https://www.urdupoint.com/sports/detail-news/live-news-3011178.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_73467_1641540536.jpg._2\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       ایشز سیریز، بین سٹوکس بولڈ ہونےکے باوجود ناٹ آؤٹ قرار\n",
      "description:  سڈنی (اردو پوائنٹ تازہ ترین اخبار ۔ 7 جنوری 2022ء ) ایشز سیریز کے چوتھے ٹیسٹ می\n",
      "permalink:   https://www.urdupoint.com/sports/detail-news/live-news-3011122.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_f4943_1641538005.jpg._2\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       آسٹریلین عدالت کا جوکووچ کو پیر سے قبل بیدخل نہ کرنیکا حکم\n",
      "description:  ٹینس  میلبورن (اردو پوائنٹ تازہ ترین اخبار ۔ 7 جنوری 2022ء ) عالمی نمبر ایک ٹین\n",
      "permalink:   https://www.urdupoint.com/sports/detail-news/live-news-3011120.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_d280c_1641537335.jpg._2\n",
      "category:    Sports\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       کورونا کا پھیلاؤ خطرناک‘ سعودیہ میں مسلسل دوسرے روز 3000 سے زائد کیسز رپورٹ\n",
      "description:  سعودی عرب  ریاض ( اُردو پوائنٹ تازہ ترین اخبار ۔ 06 جنوری 2022ء )  سعودی عرب می\n",
      "permalink:   https://www.urdupoint.com/international/news-detail/saudi-arabia/riyadh/live-news-3010330.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2020/420x350/pic_61138_1591793596.jpg._2\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       یو اے ای میں کورونا کا پھیلاؤ برقرار‘ یومیہ کیسز کی تعداد آج بھی 2600 سے نیچے نہ آئی\n",
      "description:  متحدہ عرب امارات  ابو ظہبی ( اُردو پوائنٹ تازہ ترین اخبار ۔ 06 جنوری 2022ء )  م\n",
      "permalink:   https://www.urdupoint.com/international/news-detail/uae/abu-dhabi/live-news-3010290.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_b578e_1641466646.jpg._2\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       عمان میں پاکستانیوں کیلئے روزگار کے مزید موقع پیدا ہونے کا امکان روشن ہوگیا\n",
      "description:  عمان  کراچی / مسقط ( اُردو پوائنٹ تازہ ترین اخبار ۔ 06 جنوری 2022ء ) خلیجی سلطن\n",
      "permalink:   https://www.urdupoint.com/international/news-detail/oman/muscat/live-news-3010241.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_0dfb2_1641463410.jpg._2\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       سعودیہ میں پکنک مناتے ہوئے آگ جلاتے وقت ایک بات ضرور ذہن میں رکھیں\n",
      "description:  سعودی عرب   ریاض ( اُردو پوائنٹ تازہ ترین اخبار ۔ 06 جنوری 2022ء )  سعودی عرب م\n",
      "permalink:   https://www.urdupoint.com/international/news-detail/saudi-arabia/riyadh/live-news-3010224.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_6e82b_1641462318.jpg._2\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       سعودی عرب میں گاڑیاں عنقریب ہائیڈروجن پر چلیں گی\n",
      "description:  سعودی عرب  ریاض ( اُردو پوائنٹ تازہ ترین اخبار ۔ 06 جنوری 2022ء )  سعودی عرب کی\n",
      "permalink:   https://www.urdupoint.com/international/news-detail/saudi-arabia/riyadh/live-news-3010190.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_f9b33_1641459278.jpg._2\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       دبئی ؛ گودام سے کباڑ چوری کرنے والے ایشیائی کارکن کو جیل کی سیر کرنا پڑ گئی\n",
      "description:  متحدہ عرب امارات  دبئی ( اُردو پوائنٹ تازہ ترین اخبار ۔ 06 جنوری 2022ء )  متحدہ\n",
      "permalink:   https://www.urdupoint.com/international/news-detail/uae/dubai/live-news-3010150.html\n",
      "channel:     Urdupoint\n",
      "image:       https://photo-cdn.urdupoint.com/show_img_new/daily/todayNewsLive/2022/420x350/pic_91853_1641456611.jpg._2\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       جاوید اختر کو مسلم خواتین کی فروخت کے ماسٹرمائنڈ کی حمایت مہنگی پڑگئی\n",
      "description:  بھارت  ممبئی(اُردو پوائنٹ اخبارتازہ ترین - آن لائن۔ 06 جنوری2022ء) مشہور شاعر ا\n",
      "permalink:   https://www.urdupoint.com/showbiz/news/3010433.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       سلمان خان نے محبت کا اظہار کر کے شادی سے انکار کردیا‘سومی علی\n",
      "description:  امریکی  ممبئی(اُردو پوائنٹ اخبارتازہ ترین - آن لائن۔ 06 جنوری2022ء) پاکستانی نژ\n",
      "permalink:   https://www.urdupoint.com/showbiz/news/3010434.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       اللہ کے راستے پر چلنے سے جو اطمینان اور لطف ملا ماضی کی زندگی اسکے مقابلے میں کچھ بھی نہیں ‘ رابی پیرزادہ\n",
      "description:  دنیا  لاہور (اُردو پوائنٹ اخبارتازہ ترین - آن لائن۔ 06 جنوری2022ء) سابقہ گلوکار\n",
      "permalink:   https://www.urdupoint.com/showbiz/news/3010435.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       زندگی میں مشکلات کا کا سامنا کرنا سیکھ لیا ہے ‘ حمیرا ارشد\n",
      "description:  شوبز  لاہور (اُردو پوائنٹ اخبارتازہ ترین - آن لائن۔ 06 جنوری2022ء) نامور گلوکار\n",
      "permalink:   https://www.urdupoint.com/showbiz/news/3010436.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       فیملی کی سپورٹ نہ ملتی تو کبھی بھی موجودہ مقام حاصل نہ کر سکتی ‘ ثنا جاوید\n",
      "description:  شوبز  لاہور (اُردو پوائنٹ اخبارتازہ ترین - آن لائن۔ 06 جنوری2022ء) اداکارہ ثنا \n",
      "permalink:   https://www.urdupoint.com/showbiz/news/3010437.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       انسان کو دنیا کیساتھ ساتھ آخرت میں بھی سر خرو ہونے کیلئے جدوجہد کرنی چاہیے ‘ افتخار ٹھاکر\n",
      "description:  دنیا  لاہور (اُردو پوائنٹ اخبارتازہ ترین - آن لائن۔ 06 جنوری2022ء) نامور کامیڈی\n",
      "permalink:   https://www.urdupoint.com/showbiz/news/3010438.html\n",
      "channel:     Urdupoint\n",
      "image:       \n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 24\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources10 = [\n",
    "                 ['https://www.urdupoint.com/pakistan','National-local','Urdupoint'],\n",
    "                 ['https://www.urdupoint.com/sports','Sports','Urdupoint'],\n",
    "                 ['https://www.urdupoint.com/international','International','Urdupoint'],\n",
    "                 ['https://www.urdupoint.com/showbiz','Entertainment','Urdupoint'],\n",
    "                ]\n",
    "    if Urdupoint == True:\n",
    "        print('Getting Urdupoint News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayUrduPoint = []\n",
    "        for source in sources10:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for a in soup.find('div',class_='list_hlaf_block').find_all('a'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = a['href']\n",
    "                        permalinkArrayUrduPoint.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayUrduPoint\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',{'id':'main_content'}):\n",
    "                try:\n",
    "                    detail_title = div.find('div',class_='main_bar').div.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    high_resolution_image = div.find('div',class_='news_article').find('div', class_='ac').picture.img['src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description = div.find('div',class_='main_bar').div.find('div',class_='detail_txt').a.text.replace('\\n','').replace('\\xa0','') + div.find('div',class_='main_bar').div.find('div',class_='detail_txt').text.replace('\\n','').replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='Urdu'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }  \n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources12 = [\n",
    "                 ['https://www.neonetwork.pk/international','International','NeoNetwork'],\n",
    "                 ['https://www.neonetwork.pk/blogs','Blog','NeoNetwork'],\n",
    "                 ['https://www.neonetwork.pk/entertainment','Entertainment','NeoNetwork'],\n",
    "                ]\n",
    "    if Neonews == True:\n",
    "        print('Getting Neo News...')\n",
    "        i = 1\n",
    "        permalinkArrayNeoNews = []\n",
    "        for source in sources12:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='min_height_for_large_widget'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = div.article.div.a['href']\n",
    "                        permalinkArrayNeoNews.append([permalink,source[1],source[2]])\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayNeoNews\n",
    "\n",
    "        # for detail_page_source in detail_source:\n",
    "        #     sourceHtml = requests.get(detail_page_source[0]).text\n",
    "        #     soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "        #     for div in soup.find_all('div',{'id':'main_content'}):\n",
    "        #         try:\n",
    "        #             detail_title = div.find('div',class_='main_bar').div.h1.text\n",
    "        # #             \n",
    "        #         except:\n",
    "        #             detail_title = ''\n",
    "        #             \n",
    "        #         try:\n",
    "        #             high_resolution_image = div.find('div',class_='main_bar').div.figure.img['src']\n",
    "        #                        \n",
    "        #         except:\n",
    "        #             high_resolution_image = 'no-image'\n",
    "        #         try:\n",
    "        #             date_time = time.ctime()\n",
    "        #         except:\n",
    "        #             date_time = ''\n",
    "        #         try:\n",
    "        #             detail_description = div.find('div',class_='main_bar').div.find('div',class_='detail_txt').a.text.replace('\\n','') + div.find('div',class_='main_bar').div.find('div',class_='detail_txt').text.replace('\\n','')\n",
    "        # #             print(description)\n",
    "        # #             detail_description_all = div.find('div',class_='content_detail').find_all('p')\n",
    "        # #             detail_description = ''\n",
    "        # #             for p in detail_description_all:\n",
    "        # #                 detail_description = detail_description + p.text\n",
    "        #         except:\n",
    "        #             \n",
    "        #         language='Urdu'\n",
    "        #         values = {\n",
    "        #                     'title': detail_title,\n",
    "        #                     'description': detail_description,\n",
    "        #                     'date': date_time,\n",
    "        #                     'channel': source[2],\n",
    "        #                     'image': high_resolution_image,\n",
    "        #                     'permalink': detail_page_source[0],\n",
    "        #                     'category': detail_page_source[1],\n",
    "        #                     'fetching_date': date.today(),\n",
    "        #                     'no_of_comments': 0,\n",
    "        #                     'rating': 0,\n",
    "        #                     'language':language,\n",
    "        #                     'created_at': date.today(),\n",
    "        #                     'updated_at': date.today(),\n",
    "        #                     'comments': [],\n",
    "        #                     'resgistered_views':[],\n",
    "        #                     'no_of_registered_views':0,\n",
    "        #                     'no_of_nonregistered_views':0,\n",
    "        #                     }\n",
    "        #         \n",
    "        #         print('----------------')       \n",
    "            # if(developmentMode):\n",
    "            #         data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "            #         \n",
    "            #         response = requests.post(url, data=values)\n",
    "            #         if response.status_code == 429:\n",
    "            #             time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "            #     else:\n",
    "            #         data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "            #         url = \"http://3.142.50.232:5000/api/news/all\"\n",
    "            #         response = requests.post(url, data=values)\n",
    "            #         if response.status_code == 429:\n",
    "            #             time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "\n",
    "except:\n",
    "    print('exception in ' + source[2])\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Royal News...\n",
      "total news found on listting page : 40\n",
      "\n",
      "No:          1\n",
      "title:       لاہور سمیت پنجاب کے مختلف اضلاع میں کل سے اسکول کھل جائیں گے….\n",
      "description: لاہور: (رائل نیوز) لاہور سمیت پنجاب کے مختلف اضلاع میں کل سے اسکول کھل جائیں گے۔\n",
      "permalink:   https://royalnews.tv/?p=49006\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-19.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       اومی کرون پر مسلسل نظریں مرکوز، لاک ڈاؤن کا ابھی فیصلہ نہیں کیا\n",
      "description: اسلام آبااد: (رائل نیوز) وفاقی وزیراسد عمر نے کہاہے کہ کورونا کی قسم اومی کرون \n",
      "permalink:   https://royalnews.tv/?p=49003\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-18.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       کراچی میں دہشتگردی کا بڑا منصوبہ ناکام\n",
      "description: کراچی: (رائل  نیوز) کراچی کے علاقے کورنگی میں پولیس نے دہشتگردی کا بڑا منصوبہ نا\n",
      "permalink:   https://royalnews.tv/?p=49000\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-17.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       تحریک انصاف کی سکروٹنی کمیٹی رپورٹ بارش کا پہلا قطرہ ہے\n",
      "description: لاہور: (رائل نیوز) پنجاب اسمبلی میں اپوزیشن لیڈر حمزہ شہبازشریف نے کہا ہے کہ تحر\n",
      "permalink:   https://royalnews.tv/?p=48997\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-16.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       ملک میں پی ڈی ایم کی انتشار پھیلانے کی سیاست نہیں چلے گی\n",
      "description: لاہور: (رائل نیوز) وزیراعلیٰ عثمان بزدار نے کہا ہے کہ ملک میں پی ڈی ایم کی انتشا\n",
      "permalink:   https://royalnews.tv/?p=48994\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-15.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       بلوچستان میں بارشوں سے تباہی\n",
      "description: کوئٹہ: (رائل نیوز) بلوچستان کے مکران ڈویژن میں بارشوں نے تباہی مچا دی ہے۔ گوادر \n",
      "permalink:   https://royalnews.tv/?p=48991\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-14.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       وزیراعظم سے جھوٹی تقاریر، ٹویٹس پر استعفیٰ دینے کا مطالبہ\n",
      "description: لاہور: (رائل نیوز) مسلم لیگ ن کی ترجمان نے وزیراعظم سے جھوٹی تقاریر، ٹویٹس پر اس\n",
      "permalink:   https://royalnews.tv/?p=48988\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-13.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       قوم کا پیسہ بچانے پر وزارت مواصلات مبارکباد کی مستحق\n",
      "description: اسلام آباد: (رائل نیوز) وزیراعظم عمران خان نے قوم کا پیسہ بچانے اور ن لیگ کی حک\n",
      "permalink:   https://royalnews.tv/?p=48985\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-12.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       ملک بھر میں 24 گھنٹے کے دوران کورونا سے مزید 2 اموات\n",
      "description: اسلام آباد: (ویب ڈیسک) کورونا وائرس سے 2 افراد جاں بحق ہوگئے، جس کے بعد اموات ک\n",
      "permalink:   https://royalnews.tv/?p=48982\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/635435_45612536.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       لاہور، اسلام آباد ،سمیت ملک کے مختلف شہروں میں بادل برس پڑے\n",
      "description: لاہور، اسلام آباد سمیت ملک کے مختلف شہروں میں بادل برس پڑے، آسمان سے برستی بوندو\n",
      "permalink:   https://royalnews.tv/?p=48979\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/635448_71307302.jpg\n",
      "category:    National-local\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       ٹوئٹر نے امریکی کانگریس ویمن مرجوری ٹیلر کا اکاؤنٹ مستقل طور پر بند\n",
      "description: نیویارک :سماجی رابطے کی ویب سائٹ ٹوئٹر نے امریکی کانگریس ویمن مرجوری ٹیلر کا اکا\n",
      "permalink:   https://royalnews.tv/?p=48973\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-11.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       نیدرزلینڈ: لاک ڈاؤن کیخلاف ہزاروں شہریوں کا احتجاج\n",
      "description: ایمسٹرڈیم: نیدرزلینڈ میں کورونا لاک ڈاؤن کے خلاف ہزاروں شہری سڑکوں پر نکل آئے، \n",
      "permalink:   https://royalnews.tv/?p=48970\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-10.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       سوڈان: فوجی حکومت کیخلاف مظاہروں میں ہلاکتیں\n",
      "description: خرطوم:  سوڈان میں فوجی حکومت کے خلاف مظاہروں میں ہلاکتوں کے بعد وزیراعظم عبداللہ\n",
      "permalink:   https://royalnews.tv/?p=48967\n",
      "channel:     Royalnews\n",
      "image:       https://royalnews.tv/wp-content/uploads/2022/01/1-9.jpg\n",
      "category:    International\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "exception in Royalnews\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources20 = [\n",
    "                 ['https://royalnews.tv/?cat=2','National-local','Royalnews'],\n",
    "                 ['https://royalnews.tv/?cat=3','International','Royalnews'],\n",
    "                 ['https://royalnews.tv/?cat=4','Sports','Royalnews'],\n",
    "                 ['https://royalnews.tv/?cat=5','Entertainment','Royalnews'],    \n",
    "                ]\n",
    "    if Royal == True:\n",
    "        print('Getting Royal News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayNeoNews = []\n",
    "        for source in sources20:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('article',class_='format-standard'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = article.header.h1.a['href']\n",
    "                        permalinkArrayNeoNews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "                i = 1\n",
    "        detail_source = permalinkArrayNeoNews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('article',class_='format-standard'):\n",
    "                try:\n",
    "                    detail_title = article.header.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    high_resolution_image = article.header['style'][15:-45]\n",
    "                except:\n",
    "                    high_resolution_image = 'no-image'\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description = article.find('div',class_='entry-content').p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='Urdu'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }        \n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:   ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Mashion(showbiz) News...\n",
      "total news found on listting page : 0\n",
      "total news scraped page : 0\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources23 = [\n",
    "                 ['https://mashion.pk/category/mashup/fashion/','Fashion','Mashion'],\n",
    "                 ['https://mashion.pk/category/mashup/celebrity/','Entertainment','Mashion'],    \n",
    "                ]\n",
    "    if Mashion == True:\n",
    "        counter = 0\n",
    "        print('Getting Mashion(showbiz) News...')\n",
    "        i = 1\n",
    "        permalinkArrayNeoNews = []\n",
    "        for source in sources23:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='large-4'):\n",
    "                if i<= 6:\n",
    "                    try:\n",
    "                        permalink = div.h2.a['href']\n",
    "                        permalinkArrayNeoNews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayNeoNews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find('div',{'id':'content'}):\n",
    "                try:\n",
    "                    detail_title = div.div.find('div',class_='title').h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.div.div['data-bg'][4:-1]\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description = soup.find('div', class_='wpb_wrapper').div.div.p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='Urdu'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }\n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                break\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Mangobaaz News...\n",
      "total news found on listting page : 12\n",
      "\n",
      "No:          1\n",
      "title:       Proton Pakistan Partners With Euro Oil To Launch Proton Genuine Oil\n",
      "description: Al-Haj Automotive Pvt Ltd. – the progressive assembler and distributor of Proton\n",
      "permalink:     https://www.mangobaaz.com/proton-pakistan-partners-with-euro-oil-to-launch-proton-genuine-oil\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2022/01/pririrriir.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       Premium Quality Fast Fashion That Won't Break The Bank – Dresscode’s Biggest Offer Yet!\n",
      "description: It’s here! The greatest 100% cashback offer and sale of the season. Can you imag\n",
      "permalink:     https://www.mangobaaz.com/premium-quality-fast-fashion-that-wont-break-the-bank-dresscodes-biggest-offer-yet\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/12/fdfdfdfd.png\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       Kick Starting New Year Celebrations With A Fun Recap: Celebrating Best Of 2021 – #BYE2021 #HI2022 Challenge\n",
      "description: 2022 is right around the corner and the world is gearing up to celebrate the com\n",
      "permalink:     https://www.mangobaaz.com/kick-starting-new-year-celebrations-with-a-fun-recap-celebrating-best-of-2021-bye2021-hi2022-challenge\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/12/snackvidddd.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       Dream Big, Soar High! 8 Reasons Why Women Should Apply For Parwan-e-Nisa\n",
      "description: The last two years have seen an unprecedented surge in the entrepreneurial lands\n",
      "permalink:     https://www.mangobaaz.com/dream-big-soar-high-8-reasons-why-women-should-apply-for-parwan-e-nisa\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/12/hdhdhdhdh.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       COVID Memes That Hilariously Got Pakistanis Through 2021\n",
      "description: COVID-19 spread itself around the whole globe the way a fire ravages a forest. W\n",
      "permalink:     https://www.mangobaaz.com/covid-memes-that-hilariously-got-pakistanis-through-2021\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/12/haniadananaeermbcover.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Pantene Hum Bridal Couture Week Shines Bright With Their “Hair Show”\n",
      "description: One of the biggest Pakistani fashion events recently took place in Lahore. It is\n",
      "permalink:     https://www.mangobaaz.com/pantene-hum-bridal-couture-week-shines-bright-with-their-hair-show\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/12/pantene-hair-show.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       15 Pakistani Celebrities Who Got Married/Welcomed Babies In 2021\n",
      "description: Another year has flown by with the world still under the grip of the deadly pand\n",
      "permalink:     https://www.mangobaaz.com/15-pakistani-celebrities-who-got-married-welcomed-babies-in-2021\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/12/celebrity-weddings-babies-2021.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       VicKat Got Married And Here Are ALL The Adorable Pictures\n",
      "description: Katrina Kaif and Vicky Kaushal – or VicKat, as they’re fondly called – kept thei\n",
      "permalink:     https://www.mangobaaz.com/vickat-got-married-and-here-are-all-the-adorable-pictures\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/12/VicKat-couple-goals.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Sex Education Was Trending In Pakistan. Here's What I Think About It\n",
      "description: To absolutely no one’s surprise, when Season 3 of ‘Sex Education’ dropped on Net\n",
      "permalink:     https://www.mangobaaz.com/sex-education-was-trending-in-pakistan-heres-what-i-think-about-it\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/09/mbcoversexedpak.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       Minal Khan And Ahsan Ikram's Wedding Pictures Are The Absolute Cutest\n",
      "description: Minal Khan and Ahsan Ikram’s super sweet wedding celebrations are the ‘talk of t\n",
      "permalink:     https://www.mangobaaz.com/minal-khan-and-ahsan-ikrams-wedding-pictures-are-the-absolute-cutest\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/09/Minal-and-Ahsans-wedding.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       Minal Khan & Ahsan Ikram's Mayun Makes Me Believe In Love Again\n",
      "description: Actors Minal Khan and Ahsan Ikram’s Mayun event took place last night and safe t\n",
      "permalink:     https://www.mangobaaz.com/minal-khan-ahsan-ikrams-mayun-makes-me-believe-in-love-again\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/09/0001-7517007833_20210909_151116_0000.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       Minal Khan And Ahsan Ikram's Wedding Festivities Have Begun & MashAllah\n",
      "description: The long-awaited wedding festivities for Minal Khan and Ahsan Ikram have officia\n",
      "permalink:     https://www.mangobaaz.com/minal-khan-and-ahsan-ikrams-wedding-festivities-have-begun-mashallah\n",
      "channel:     Mangobaaz\n",
      "image:       https://cdn.mangobaaz.com/wp-content/uploads/2021/09/Untitled-design-1.jpg\n",
      "category:    Blog\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 12\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources24 = [\n",
    "                 ['https://www.mangobaaz.com/recents','Blog','Mangobaaz'],\n",
    "                 ['https://www.mangobaaz.com/category/showsha','Blog','Mangobaaz'],    \n",
    "                ]\n",
    "    if Mangobaaz == True:\n",
    "        print('Getting Mangobaaz News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayMangobaaz = []\n",
    "        for source in sources24:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "\n",
    "            for a in soup.find_all('a',class_='block focus:outline-none'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink ='https://www.mangobaaz.com' + a['href']\n",
    "                        permalinkArrayMangobaaz.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayMangobaaz\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div', class_='leading-normal'):\n",
    "                try:\n",
    "                    detail_title = div.main.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.div['style'][21:-2]\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description = div.main.article.p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }  \n",
    "\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter)) \n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting HelloPakistanMagazine News...\n",
      "total news found on listting page : 12\n",
      "\n",
      "No:          1\n",
      "title:       SHAADI SEASON 101: DID YOU KNOW THERE ARE 8 HENNA ‘CUISINES’?\n",
      "description:  Henna, also referred to as mehndi in the Indian subcontinent is a type of body \n",
      "permalink:     https://www.hellopakistanmag.com/03-Jan-2022/shaadi-season-101-did-you-know-there-are-8-henna-cuisines\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2022-01-03/shaadi-season-101-did-you-know-there-are-8-henna-cuisines-1641223010-1581.jpeg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       2021’S MOST LOVED BRIDAL LOOKS\n",
      "description:  Despite the ongoing pandemic, many celebrities went forward with their weddings\n",
      "permalink:     https://www.hellopakistanmag.com/31-Dec-2021/2021-s-most-loved-bridal-looks\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2021-12-31/2021-s-most-loved-bridal-looks-1640905841-5782.jpeg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       All eyes on Pantene’s Hair Show at this year’s #PHBCW\n",
      "description: The 19th Pantene Hum Bridal Couture Week took place in a very chilly Lahore from\n",
      "permalink:     https://www.hellopakistanmag.com/25-Dec-2021/all-eyes-on-pantene-s-hair-show-at-this-year-s-phbcw\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2021-12-25/all-eyes-on-pantene-s-hair-show-at-this-year-s-phbcw-1640450337-5771.jpeg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       RECONNECTING WITH OUR ROOTS: INTERNATIONAL DESIGNER ALI HAIDER DAWNS HIS CREATIVE VISION IN PAKISTAN\n",
      "description:  The clothing I created in Pakistan will celebrate the heritage and craftsmanshi\n",
      "permalink:     https://www.hellopakistanmag.com/21-Dec-2021/reconnecting-with-our-roots-international-designer-ali-haider-dawns-his-creative-vision-in-pakistan\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2021-12-21/reconnecting-with-our-roots-international-designer-ali-haider-dawns-his-creative-vision-in-pakistan-1640067333-1248.jpeg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       WHAT IS 'A REAL MODEL'? FT. DUBAI-BASED PAKISTANI MODEL HAMZA SALAH\n",
      "description:  I’m a real model - by that I mean I’m a real person. My life is all about inspi\n",
      "permalink:     https://www.hellopakistanmag.com/16-Dec-2021/what-is-a-real-model-ft-dubai-based-pakistani-model-hamza-salah\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2021-12-16/what-is-a-real-model-ft-dubai-based-pakistani-model-hamza-salah-1639667416-3437.jpeg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Khaadi is Wild About you and the New Experience Hub proves it\n",
      "description: Karachi, 3 December: Khaadi stuns everyone once again by taking the lead as Paki\n",
      "permalink:     https://www.hellopakistanmag.com/15-Dec-2021/khaadi-is-wild-about-you-and-the-new-experience-hub-proves-it\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2021-12-15/khaadi-is-wild-about-you-and-the-new-experience-hub-proves-it-1639566158-2785.jpeg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       IMTISAL ZAFAR: THE MAN, THE MYTH, THE LEGEND REVAMPING ART & E-MUSIC\n",
      "description:  A proponent of artistic revamping, Imtisal Zafar is renowned for three power pa\n",
      "permalink:     https://www.hellopakistanmag.com/04-Jan-2022/imtisal-zafar-the-man-the-myth-the-legend-revamping-art-e-music\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2022-01-04/imtisal-zafar-the-man-the-myth-the-legend-revamping-art-e-music-1641307632-1700.jpeg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       HIP HOP IN PAKISTAN: WHAT’S POPPIN’?\n",
      "description:  Through the 2000s, the rap songs were mostly in English and Punjabi. We saw ban\n",
      "permalink:     https://www.hellopakistanmag.com/03-Jan-2022/hip-hop-in-pakistan-what-s-poppin\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2022-01-03/hip-hop-in-pakistan-what-s-poppin-1641224870-4778.jpeg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       SEVEN CELEBRITY STATEMENTS SWORN TO BACKLASH IN 2021\n",
      "description:  Surely, fans pick up everything celebrities say, but sometimes the response sti\n",
      "permalink:     https://www.hellopakistanmag.com/02-Jan-2022/seven-celebrity-statements-sworn-to-backlash-in-2021\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2022-01-02/seven-celebrity-statements-sworn-to-backlash-in-2021-1641145704-2262.jpeg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       TO THE FORGOTTEN PARTITION: HUM TV’S KHWAB TOOT JATE HAIN HIGHLIGHTS EAST AND WEST PAKISTAN’S\n",
      "description:  Stories like Dastaan and Aangan have made waves on television, educating the ma\n",
      "permalink:     https://www.hellopakistanmag.com/02-Jan-2022/to-the-forgotten-partition-hum-tv-s-khwab-toot-jate-hain-highlights-east-and-west-pakistan-s\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2022-01-02/to-the-forgotten-partition-hum-tv-s-khwab-toot-jate-hain-highlights-east-and-west-pakistan-s-1641144230-2291.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       A - LISTERS SPILL THE TEA ON MOMINA’S MIXED PLATE\n",
      "description:  The latest episodes of Momina’s Mixed Plate featured some of the hottest supers\n",
      "permalink:     https://www.hellopakistanmag.com/31-Dec-2021/a-listers-spill-the-tea-on-momina-s-mixed-plate\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2021-12-31/a-listers-spill-the-tea-on-momina-s-mixed-plate-1640964915-7236.jpg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       BREAKTHROUGH TV DRAMA DEBUTS IN 2021\n",
      "description:  2021 was exceptional in terms of the drama we got to watch; we had a wide range\n",
      "permalink:     https://www.hellopakistanmag.com/31-Dec-2021/breakthrough-tv-drama-debuts-in-2021\n",
      "channel:     HelloPakistanMagazine\n",
      "image:       https://www.hellopakistanmag.com/digital_images/large/2021-12-31/breakthrough-tv-drama-debuts-in-2021-1640907115-1418.jpeg\n",
      "category:    Entertainment\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 12\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources25 = [\n",
    "                 ['https://www.hellopakistanmag.com/fashion','Fashion','HelloPakistanMagazine'],\n",
    "                 ['https://www.hellopakistanmag.com/entertainment','Entertainment','HelloPakistanMagazine'],\n",
    "                ]\n",
    "    if Hellopakistanmag == True:\n",
    "        print('Getting HelloPakistanMagazine News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayHelloPakistanMag = []\n",
    "        for source in sources25:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='col-md-6'):\n",
    "                if i<= 6:\n",
    "                    try:\n",
    "                        permalink = div.find('div',class_='featured-content').a['href']\n",
    "                        permalinkArrayHelloPakistanMag.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayHelloPakistanMag\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='article-content'):\n",
    "                try:\n",
    "                    detail_title = div.h3.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.find('div',class_='article-image').img['src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text.replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }  \n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sunday News...\n",
      "total news found on listting page : 37\n",
      "\n",
      "No:          1\n",
      "title:       Kiran Malik Is An Epitome Of Elegance In This Color-Block Attire!\n",
      "description: The glamorous, Kiran Malik makes bold style statements wearing a yellow and whit\n",
      "permalink:     https://sunday.com.pk/fashion/look-of-the-day/kiran-malik-is-an-epitome-of-elegance-in-this-color-block-attire/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2022/01/271340106_434790488240648_6227589212986759807_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          2\n",
      "title:       Style Icon of the Week: Working Mom Amna Malik\n",
      "description: While Amna Malik made her silver screen debut in 2017 with Kitni Girhain Baaki H\n",
      "permalink:     https://sunday.com.pk/fashion/style-icon-of-the-week-working-mom-amna-malik/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2022/01/127113220_3429600257075901_5589972874098880441_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          3\n",
      "title:       Funky Is The New Cool | Fashion Designer Kamiar Rokni Paves The Way For Fashion Fusion In 2022\n",
      "description: In the realm of traditional designs, funky trends stand out and make their mark.\n",
      "permalink:     https://sunday.com.pk/fashion/funky-is-the-new-cool-famous-fashion-designer-kamiar-rokni-becomes-the-trendsetter/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2022/01/269894664_857183431616451_8081503495998315349_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          4\n",
      "title:       Fashion Face-Off | Our Favorite Lolly Wood Actresses In their Wedding Best\n",
      "description: Everywhere we look, there’s a new and gorgeous outfit that emerges being adorned\n",
      "permalink:     https://sunday.com.pk/fashion/who-wore-it-best/fashion-face-off-our-favorite-lolly-wood-actresses-in-their-wedding-best/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2022/01/Blank-4-Grids-Collage-1-1600x1600.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          5\n",
      "title:       Bridal Fashion Bridging Differences Between Borders\n",
      "description: We’re all familiar with the exquisite work of Indian designer Sabyasachi Mukherj\n",
      "permalink:     https://sunday.com.pk/fashion/trends-fashion/bridal-fashion-bridging-differences-between-borders/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/109307083_2459904120977847_2209705241104072676_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          6\n",
      "title:       Day To Play: 7 Foolproof Hairstyles to Last You from AM to PM\n",
      "description: Hair is an integral part of your complete look – it can make or break any outfit\n",
      "permalink:     https://sunday.com.pk/beauty/hair/day-to-play-7-foolproof-hairstyles-to-last-you-from-am-to-pm/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/Blank-4-Grids-Collage-1-1600x1600.png\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          7\n",
      "title:       Style Icon of the Week: Working Mom Amna Malik\n",
      "description: While Amna Malik made her silver screen debut in 2017 with Kitni Girhain Baaki H\n",
      "permalink:     https://sunday.com.pk/fashion/style-icon-of-the-week-working-mom-amna-malik/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2022/01/127113220_3429600257075901_5589972874098880441_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          8\n",
      "title:       Funky Is The New Cool | Fashion Designer Kamiar Rokni Paves The Way For Fashion Fusion In 2022\n",
      "description: In the realm of traditional designs, funky trends stand out and make their mark.\n",
      "permalink:     https://sunday.com.pk/fashion/funky-is-the-new-cool-famous-fashion-designer-kamiar-rokni-becomes-the-trendsetter/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2022/01/269894664_857183431616451_8081503495998315349_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          9\n",
      "title:       Black is the New Black | A Round-Up of Winter Shaadi Looks With The Classic Colour\n",
      "description: Gone are the days when black was considered an ominous colour for special occasi\n",
      "permalink:     https://sunday.com.pk/fashion/black-is-the-new-black-a-round-up-of-winter-shaadi-looks-with-the-classic-colour/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/black-collage-1.png\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          10\n",
      "title:       The Viral Bride Who Wore A Fusion Of Indian And Pakistani Designers: Her Looks Dissected\n",
      "description: Altish Yousaf got married recently and her exceptional looks on all her wedding \n",
      "permalink:     https://sunday.com.pk/beauty/hair/the-viral-bride-who-wore-a-fusion-of-indian-and-pakistani-designers-her-looks-dissected/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/Untitled-collage-8-1-1-1600x1600.png\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          11\n",
      "title:       Fashion But Make It Fusion\n",
      "description: We love our ethnic silhouettes at Sunday Times, but sometimes it’s fun to switch\n",
      "permalink:     https://sunday.com.pk/fashion/fashion-but-make-it-fusion/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/imgescoverfinal-1600x1305.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          12\n",
      "title:       Khaadi Blazes Ahead with Their Innovative Vision for Fashion Retail\n",
      "description: Let’s take a journey through their experience hub.Karachi, 3 December: Step into\n",
      "permalink:     https://sunday.com.pk/fashion/khaadi-blazes-ahead-with-their-innovative-vision-for-fashion-retail/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/Screenshot-2021-12-10-at-20.03.44-1600x1236.png\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          13\n",
      "title:       Style Icon of the Week: Working Mom Amna Malik\n",
      "description: While Amna Malik made her silver screen debut in 2017 with Kitni Girhain Baaki H\n",
      "permalink:     https://sunday.com.pk/fashion/style-icon-of-the-week-working-mom-amna-malik/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2022/01/127113220_3429600257075901_5589972874098880441_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          14\n",
      "title:       Best Dressed Celebrities Of 2021 Making A Statement!\n",
      "description: There have been numerous events throughout the year and our glamorous celebritie\n",
      "permalink:     https://sunday.com.pk/fashion/best-dressed/best-dressed-celebrities-of-2021-making-a-statement/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/Blank-3-Grids-Collage-2-1600x1600.png\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          15\n",
      "title:       Sonya Hussyn Is A Dream In This Jaw-Dropping Mirror-Work Lehenga\n",
      "description: A plethora of colors, exquisite mirror work, and ravishing flair, Nomi Ansari do\n",
      "permalink:     https://sunday.com.pk/fashion/look-of-the-day/sonya-hussain-looking-like-a-dream-in-this-mirror-work-lehnga-choli-by-nomi-ansari/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/270295742_968746660685951_1814187317624277917_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          16\n",
      "title:       Black is the New Black | A Round-Up of Winter Shaadi Looks With The Classic Colour\n",
      "description: Gone are the days when black was considered an ominous colour for special occasi\n",
      "permalink:     https://sunday.com.pk/fashion/black-is-the-new-black-a-round-up-of-winter-shaadi-looks-with-the-classic-colour/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/black-collage-1.png\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          17\n",
      "title:       The Viral Bride Who Wore A Fusion Of Indian And Pakistani Designers: Her Looks Dissected\n",
      "description: Altish Yousaf got married recently and her exceptional looks on all her wedding \n",
      "permalink:     https://sunday.com.pk/beauty/hair/the-viral-bride-who-wore-a-fusion-of-indian-and-pakistani-designers-her-looks-dissected/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/Untitled-collage-8-1-1-1600x1600.png\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          18\n",
      "title:       9 Of The Best Instagram Winter Trends 2021| Baby, It’s Cold Outside\n",
      "description: As we approach winter season in full swing, it’s time to start looking for some \n",
      "permalink:     https://sunday.com.pk/fashion/baby-its-cold-outside/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/covertrendinfg-1600x667.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          19\n",
      "title:       Fashion But Make It Fusion\n",
      "description: We love our ethnic silhouettes at Sunday Times, but sometimes it’s fun to switch\n",
      "permalink:     https://sunday.com.pk/fashion/fashion-but-make-it-fusion/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/imgescoverfinal-1600x1305.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          20\n",
      "title:       Asifa & Nabeel Set HUM Bridal Week Stage On Fire With Their Sizzling Designs\n",
      "description: As we witnessed a plethora of designers showcasing their latest and best clothin\n",
      "permalink:     https://sunday.com.pk/fashion/fashion-week/asifa-nabeel-set-the-hum-bridal-week-stage-on-fire-with-their-sizzling-designs/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/asifa-and-nabeel-1600x1600.png\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          21\n",
      "title:       These Lux Style Awards Portraits Took Our Breath Away\n",
      "description: Photographer Jaffer Hasan manages to always capture a behind-the-scenes moment l\n",
      "permalink:     https://sunday.com.pk/fashion/these-lux-style-awards-portraits-took-our-breath-away/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/01/3BE3130C-1BAF-4598-95F6-83994AABFCE7.jpeg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          22\n",
      "title:       FAS Design Studio’s star studded fashion showcase at PLBW\n",
      "description: Its been over a week since the PFDC Loreal Bridal Week concluded and this instal\n",
      "permalink:     https://sunday.com.pk/fashion/fas-design-studios-star-studded-fashion-showcase-at-plbw/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2019/10/goup-resize.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          23\n",
      "title:       PLBW Day 3 : The Finale\n",
      "description: After 2 exceptional days of fashion, expectations were high for Day 3 of PLBW. A\n",
      "permalink:     https://sunday.com.pk/fashion/plbw-day-3-the-finale/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2019/09/day-3.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          24\n",
      "title:       Asma Aslam Showcases her Debut Collection at PLBW 2019\n",
      "description: All the fashionistas in town have their calendars full this week because Pakista\n",
      "permalink:     https://sunday.com.pk/fashion/asma-aslam-showcases-her-debut-collection-at-plbw-2019/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2019/09/asma-aslam.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          25\n",
      "title:       PLBW Day 2 : Hits & Misses\n",
      "description: Day 2 of PLBW included the biggest names in the fashion industry such as Kamiar \n",
      "permalink:     https://sunday.com.pk/fashion/plbw-day-2-hits-misses/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2019/09/day2-2.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          26\n",
      "title:       Kiran Malik Is An Epitome Of Elegance In This Color-Block Attire!\n",
      "description: The glamorous, Kiran Malik makes bold style statements wearing a yellow and whit\n",
      "permalink:     https://sunday.com.pk/fashion/look-of-the-day/kiran-malik-is-an-epitome-of-elegance-in-this-color-block-attire/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2022/01/271340106_434790488240648_6227589212986759807_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          27\n",
      "title:       Style Icon of the Week: Working Mom Amna Malik\n",
      "description: While Amna Malik made her silver screen debut in 2017 with Kitni Girhain Baaki H\n",
      "permalink:     https://sunday.com.pk/fashion/style-icon-of-the-week-working-mom-amna-malik/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2022/01/127113220_3429600257075901_5589972874098880441_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          28\n",
      "title:       Sonya Hussyn Is A Dream In This Jaw-Dropping Mirror-Work Lehenga\n",
      "description: A plethora of colors, exquisite mirror work, and ravishing flair, Nomi Ansari do\n",
      "permalink:     https://sunday.com.pk/fashion/look-of-the-day/sonya-hussain-looking-like-a-dream-in-this-mirror-work-lehnga-choli-by-nomi-ansari/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/270295742_968746660685951_1814187317624277917_n.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          29\n",
      "title:       Taiba Malik is Divine in All Black\n",
      "description: What’s 2 things Sunday loves? Misha Lakhani and an all black outfit. And that’s \n",
      "permalink:     https://sunday.com.pk/fashion/taiba-malik-is-divine-in-all-black/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2019/11/Lotd-web.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          30\n",
      "title:       Sonya Hussyn is Turning Up the Heat this Winter!\n",
      "description: GIRRLLLLLL, that’s one hot look! From the ribbed crop top (MAMMA MIA, THOSE ABS!\n",
      "permalink:     https://sunday.com.pk/fashion/sonya-hussyn-is-turning-up-the-heat-this-winter/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2019/11/5EB47B12-961B-443E-BB5A-BE4D4606A187.jpeg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          31\n",
      "title:       Ayesha Omar is bringing the HEAT in this Shehla Chatoor Number\n",
      "description: Wedding season is upon us and we’re already getting look after killer look from \n",
      "permalink:     https://sunday.com.pk/fashion/look-of-the-day/ayesha-omar-is-bringing-the-heat-in-this-shehla-chatoor-number/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2019/11/ayesga-omer-web.jpg\n",
      "category:    Fashion\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          32\n",
      "title:       How To Stay Cosy In This Chilly Weather With The Best Hot Chocolate Options In Lahore\n",
      "description: Winter and hot chocolate are a combination that is irreplaceable. There is somet\n",
      "permalink:     https://sunday.com.pk/lifestyle/food/how-to-stay-cosy-in-this-chilly-weather-with-the-best-hot-chocolate-options-in-lahore/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/Untitled-collage-6-1600x1600.png\n",
      "category:    Gourmet\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          33\n",
      "title:       8 Picks From The World Of Fashion, Food And Lifestyle Sunday Editors Are Hooked On\n",
      "description: Want to know what Sunday Editors are currently hooked on these days? We’re gushi\n",
      "permalink:     https://sunday.com.pk/editors-pick/8-picks-from-the-world-of-fashion-food-and-lifestlye-sunday-editors-are-hooked-on/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/12/WhatsApp-Image-2021-12-21-at-4.48.30-PM.jpeg\n",
      "category:    Gourmet\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          34\n",
      "title:       “Gaia” Japanese Fine-Dining Restaurant Opens in Islamabad Written by Humay Waseeem\n",
      "description: Historically, Japanese restaurants have been a novelty in Islamabad, but in the \n",
      "permalink:     https://sunday.com.pk/lifestyle/food/gaia-japanese-fine-dining-restaurant-opens-in-islamabad-written-by-humay-waseeem/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/08/QAP_3660-1.jpg\n",
      "category:    Gourmet\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          35\n",
      "title:       Our Fav New Japanese Food Joint – Saku Hana\n",
      "description: Let’s address the talk of the town today – Saku Hana, the new Japanese style\n",
      "res\n",
      "permalink:     https://sunday.com.pk/lifestyle/our-fav-new-japanese-food-joint-saku-hana/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/04/Saku-hana-web.jpg\n",
      "category:    Gourmet\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          36\n",
      "title:       Here’s What’s New at ETH\n",
      "description: Every Lahori and Islooite knows of, loves and frequently visits English Tea Hous\n",
      "permalink:     https://sunday.com.pk/lifestyle/heres-whats-new-at-eth/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/02/ETH-web.jpg\n",
      "category:    Gourmet\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "\n",
      "No:          37\n",
      "title:       Bakery delights by Chashni\n",
      "description: After introducing mithai in the true traditional form representing our culture a\n",
      "permalink:     https://sunday.com.pk/lifestyle/bakery-delights-by-chashni/\n",
      "channel:     SundayNews\n",
      "image:       https://sunday.com.pk/wp-content/uploads/2021/02/Chashni-web.jpg\n",
      "category:    Gourmet\n",
      "-----------------------------------------------\n",
      "inserted in DB\n",
      "-----------------------------------------------\n",
      "total news scraped page : 37\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources24 = [\n",
    "                 ['https://sunday.com.pk/category/fashion/','Fashion','SundayNews'],\n",
    "                 ['https://sunday.com.pk/category/fashion/fashion-news/','Fashion','SundayNews'],\n",
    "                 ['https://sunday.com.pk/category/fashion/best-dressed/','Fashion','SundayNews'],\n",
    "                 ['https://sunday.com.pk/category/fashion/fashion-shoot/','Fashion','SundayNews'],\n",
    "                 ['https://sunday.com.pk/category/fashion/fashion-week/','Fashion','SundayNews'],\n",
    "                 ['https://sunday.com.pk/category/fashion/look-of-the-day/','Fashion','SundayNews'],\n",
    "                 ['https://sunday.com.pk/category/fashion/style-jury/','Fashion','SundayNews'],\n",
    "                 ['https://sunday.com.pk/category/lifestyle/food/','Gourmet','SundayNews'],\n",
    "                ]\n",
    "    if Sunday == True:\n",
    "        print('Sunday News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArraySundaynews = []\n",
    "        for source in sources24:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='post'):\n",
    "                if i<= 6:\n",
    "                    try:\n",
    "                        permalink = div.find('div',class_='row').find('div',class_='medium-7').div.find('div',class_='post-title').h5.a['href']\n",
    "                        permalinkArraySundaynews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArraySundaynews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('article'):\n",
    "                try:\n",
    "                    detail_title = article.find('div',class_='post-title-container').header.h1.text.replace('\\t','').replace('\\n','')   \n",
    "                except:\n",
    "                    detail_title = 'pakteki'\n",
    "                try:\n",
    "                    high_resolution_image = article.find('div',class_='thb-article-featured-image').img['data-src']\n",
    "\n",
    "                except:\n",
    "                    high_resolution_image = 'pakteki'\n",
    "\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = article.find('div',class_='thb-post-share-container').find('div',class_='post-content-container').find('div',class_='post-content').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text.replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = 'pakteki'\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }  \n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:80])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources27 = [\n",
    "        [\"https://www.zaiqa.com/recipes/pakistani/\",\"Gourmet\",\"Zaiqatv\"],\n",
    "        [\"https://www.zaiqa.com/recipes/sweets-desserts/\",\"Gourmet\",\"Zaiqatv\"],\n",
    "        [\"https://www.zaiqa.com/recipes/masala-tv/\",\"Gourmet\",\"Zaiqatv\"],\n",
    "        [\"https://www.zaiqa.com/recipes/indian/\",\"Gourmet\",\"Zaiqatv\"],\n",
    "        [\"https://www.zaiqa.com/recipes/zaiqa-tv/\",\"Gourmet\",\"Zaiqatv\"],\n",
    "    ]\n",
    "    if Zaiqa == True:\n",
    "        print('Getting Zaiqa News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayZaiqa = []\n",
    "        for source in sources27:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for li in soup.find_all('li',class_='group'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = source[0][:-19] + li.a['href']\n",
    "                        permalinkArrayZaiqa.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayZaiqa\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for a in detail_source:\n",
    "            print(a)\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='recipes'):\n",
    "                try:\n",
    "                    detail_title = div.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = ''\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='recipebox').ul.find_all('li')\n",
    "                    detail_description_ingredients = ''\n",
    "                    for li in detail_description_all:\n",
    "                        detail_description_ingredients = detail_description_ingredients + ' ' + li.text.replace('\\xa0','')\n",
    "                    detail_description = div.find('div',class_='recipebox').h2.text + ':' + detail_description_ingredients\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }  \n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:20])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources16 = [\n",
    "        [\"https://www.city42.tv/district-government\",\"National-local\",\"city42\"],\n",
    "        [\"https://www.city42.tv/blog\",\"Blog\",\"city42\"],\n",
    "        [\"https://www.city42.tv/lahore\",\"National-local\",\"city42\"],\n",
    "        [\"https://www.city42.tv/government-of-punjab\",\"National-local\",\"city42\"],     \n",
    "    ] \n",
    "    if City42 == True:\n",
    "        print('Getting Mashion(showbiz) News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayCity42 = []\n",
    "        for source in sources16:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='row'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = div.div.article.find('div',class_='zm-post-dis').div.h2.a['href']\n",
    "                        permalinkArrayCity42.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayCity42\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='recipes'):\n",
    "                try:\n",
    "                    detail_title = div.h1.text\n",
    "\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = ''\n",
    "\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='articles').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text.replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }  \n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:20])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "'(' was never closed (Temp/ipykernel_14292/2399686019.py, line 102)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"C:\\Users\\idenb\\AppData\\Local\\Temp/ipykernel_14292/2399686019.py\"\u001b[1;36m, line \u001b[1;32m102\u001b[0m\n\u001b[1;33m    print('total news scraped page : ' + str(counter)\u001b[0m\n\u001b[1;37m         ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m '(' was never closed\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sources28 = [\n",
    "                  ['https://www.urdunews.com/sections/%D9%BE%D8%A7%DA%A9%D8%B3%D8%AA%D8%A7%D9%86','National-local','Urdunews'],\n",
    "                  ['https://www.urdunews.com/sections/%D8%AF%D9%86%DB%8C%D8%A7','International','Urdunews'],\n",
    "                  ['https://www.urdunews.com/sections/%D8%B4%D9%88%D8%A8%D8%B2','Entertainment','Urdunews'],\n",
    "                  ['https://www.urdunews.com/sections/%DA%A9%DA%BE%DB%8C%D9%84','Sports','Urdunews']\n",
    "                 ]\n",
    "    if Urdunews == True:\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        print('Getting Urdu News...')\n",
    "        permalinkArrayUrdunews = []\n",
    "        for source in sources28:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='column'):\n",
    "                if i<=6:\n",
    "                    try:\n",
    "                        permalink = 'https://www.urdunews.com' + div.article.find('div',class_='article-item-title').h5.a['href']\n",
    "                        permalinkArrayUrdunews.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "                i = 1\n",
    "        detail_source = permalinkArrayUrdunews\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('article'):\n",
    "                try:\n",
    "                    detail_title = article.find('div',class_='entry-title').h1.text   \n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = article.find('div',class_='entry-media').img['src']  \n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = article.find('div',class_='field-items').div.find_all('h5')\n",
    "                    detail_description = ''\n",
    "                    for h5 in detail_description_all:\n",
    "                        detail_description = detail_description + h5.text.replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }  \n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:20])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter)\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources29 = [\n",
    "                  ['https://www.samaa.tv/urdu/pakistan/','Pakistan','SamaUrdu'],\n",
    "                  ['https://www.samaa.tv/urdu/international/','International','SamaUrdu'],\n",
    "                  ['https://www.samaa.tv/urdu/entertainment/','Entertainment','SamaUrdu'],\n",
    "                  ['https://www.samaa.tv/urdu/sports/','Sports','SamaUrdu'],\n",
    "                  ['https://www.samaa.tv/urdu/tag/coronavirus/','Health','SamaUrdu']\n",
    "                 ]\n",
    "    if Sama == True:\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArraySama = []\n",
    "        for source in sources28:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='column'):\n",
    "                if i<=6:\n",
    "                    try:\n",
    "                        permalink = 'https://www.urdunews.com' + div.article.find('div',class_='article-item-title').h5.a['href']\n",
    "                        permalinkArraySama.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArraySama\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('article'):\n",
    "                try:\n",
    "                    detail_title = article.find('div',class_='entry-title').h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = article.find('div',class_='entry-media').img['src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = article.find('div',class_='field-items').div.find_all('h5')\n",
    "                    detail_description = ''\n",
    "                    for h5 in detail_description_all:\n",
    "                        detail_description = detail_description + h5.text.replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }  \n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:20])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources30 = [\n",
    "                  ['https://jang.com.pk/category/latest-news/national','National-local','Jangnews'],\n",
    "                  ['https://jang.com.pk/category/latest-news/world','International','Jangnews'],\n",
    "                  ['https://jang.com.pk/category/latest-news/entertainment','Entertainment','Jangnews'],\n",
    "                  ['https://jang.com.pk/category/latest-news/sports','Sports','Jangnews'],\n",
    "                  ['https://jang.com.pk/category/latest-news/health-science','Health','Jangnews']\n",
    "                 ]\n",
    "    if Jang == True:\n",
    "        print('Getting Jang(showbiz) News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayJang = []\n",
    "        for source in sources30:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for li in soup.find_all('li'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = li.find('div',class_='main-heading').a['href']\n",
    "                        print(permalink)\n",
    "                        permalinkArrayJang.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayJang\n",
    "        print(detail_source)\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='detail-right'):\n",
    "                try:\n",
    "                    detail_title = div.div.h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.find('div',class_='detail-content').find('div',class_='description-area').find('div',class_='detail_view_content').div.figure.img['src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='detail-content').find('div',class_='description-area').find('div',class_='detail_view_content').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text.replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1\n",
    "                language='Urdu'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }  \n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:20])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if False:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources33 = [\n",
    "                 ['https://tribune.com.pk/life-style/art-books','Fashion','Express Tribune'],\n",
    "                 ['https://tribune.com.pk/life-style/fashion','Fashion','Express Tribune'],\n",
    "                 ['https://tribune.com.pk/life-style/film','Entertainment','Express Tribune'],\n",
    "                 ['https://tribune.com.pk/life-style/bollywood','Entertainment','Express Tribune'],\n",
    "                 ['https://tribune.com.pk/life-style/gossip','Entertainment','Express Tribune'],\n",
    "                 ['https://tribune.com.pk/life-style/theatre','Entertainment','Express Tribune'],\n",
    "                 ['https://tribune.com.pk/life-style/tv','Entertainment','Express Tribune'],\n",
    "\n",
    "                ]\n",
    "    if Tribune == True:\n",
    "        print('Getting Tribune Express News...')\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        permalinkArrayTribune = []\n",
    "        for source in sources33:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for li in soup.find('ul',class_='listing-page').find_all('li'):\n",
    "                if i <= 6:        \n",
    "                    try:\n",
    "                        permalink = li.find('div',class_='row').find('div',class_='col-md-8').find('div',class_='horiz-news3-caption').a['href']\n",
    "                        permalinkArrayTribune.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayTribune\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='maincontent-customwidth'):\n",
    "                try:\n",
    "                    detail_title = div.find('div',class_='story-box-section').h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = div.find('div',class_='story-box-section').find('div',class_='mainstorycontent-parent').div.div.find('span',class_='top-big-img').div.div.div.img['data-src']\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.find('div',class_='story-box-section').find('div',class_='mainstorycontent-parent').div.div.find('span',class_='story-text').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text.replace('\\xa0','')\n",
    "                except:\n",
    "                    detail_description = detail_page_source[0]\n",
    "                counter = counter + 1\n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                }  \n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:20])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources36 = [\n",
    "                ['https://theislamicinformation.com/news/page/','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/2','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/3','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/4','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/5','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/6','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/7','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/8','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/9','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/10','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/11','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/12','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/13','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/14','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/15','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/16','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/17','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/18','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/19','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/20','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/21','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/22','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/23','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/24','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/25','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/26','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/27','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/28','Islam','The Islamic Information'],\n",
    "                ['https://theislamicinformation.com/news/page/29','Islam','The Islamic Information'],\n",
    "    ]\n",
    "    if Islamicinfoenglish == True:\n",
    "        i = 1\n",
    "        counter = 0\n",
    "        print('Getting Islamic info english...')\n",
    "        permalinkArrayIslamicInformation = []\n",
    "        for source in sources36:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for article in soup.find_all('article', class_='jeg_post'):\n",
    "                if i <= 6:\n",
    "                    try:\n",
    "                        permalink = article.find('div',class_='jeg_thumb').a['href']\n",
    "                        permalinkArrayIslamicInformation.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1\n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayIslamicInformation\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='jeg_content'):\n",
    "                try:\n",
    "                    detail_title = div.div.find('div',class_='entry-header').h1.text\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "\n",
    "                try:\n",
    "                    high_resolution_image = div.div.find('div',class_='jeg_main_content').find('div',class_='featured_image').a.div.img['data-src']\n",
    "                except:\n",
    "                    high_resolution_image = 'Pakteki'\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_all = div.div.find('div',class_='jeg_main_content').div.find('div',class_='entry-content').find('div',class_='content-inner').find_all('p')\n",
    "                    detail_description = ''\n",
    "                    for p in detail_description_all:\n",
    "                        detail_description = detail_description + p.text\n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1    \n",
    "                language='English'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:20])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    sources37 = [\n",
    "                ['https://islamqa.info/ur/latest','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=2','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=3','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=4','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=5','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=6','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=7','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=8','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=9','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=10','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=11','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=12','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=13','Islam','Islamicqa'],\n",
    "                ['https://islamqa.info/ur/latest?page=14','Islam','Islamicqa'],\n",
    "                ]\n",
    "    if Islamicinfourdu == True:\n",
    "        counter = 0\n",
    "        print('Getting islamQA News...')\n",
    "        i = 1\n",
    "        permalinkArrayIslamicInformation = []\n",
    "        for source in sources37:\n",
    "            sourceHtml = requests.get(source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for a in soup.find_all('a', class_='card post-card'):\n",
    "                if i<=6:\n",
    "                    try:\n",
    "                        permalink = a['href']\n",
    "                        permalinkArrayIslamicInformation.append([permalink,source[1],source[2]])\n",
    "                        counter = counter + 1 \n",
    "                    except:\n",
    "                        permalink = ''\n",
    "                    i = i + 1\n",
    "            i = 1\n",
    "        detail_source = permalinkArrayIslamicInformation\n",
    "        if developmentMode:\n",
    "            print('total news found on listting page : ' + str(counter))\n",
    "            df = pd.DataFrame(detail_source)\n",
    "            df\n",
    "        counter = 0\n",
    "        for detail_page_source in detail_source:\n",
    "            sourceHtml = requests.get(detail_page_source[0]).text\n",
    "            soup = BeautifulSoup(sourceHtml, 'lxml')\n",
    "            for div in soup.find_all('div',class_='single_fatwa'):\n",
    "                try:\n",
    "                    detail_title = div.find('div',class_='single-layout__title').h1.text.replace('\\n','')\n",
    "                except:\n",
    "                    detail_title = ''\n",
    "                try:\n",
    "                    high_resolution_image = 'Pakteki'\n",
    "                except:\n",
    "                    high_resolution_image = ''\n",
    "                try:\n",
    "                    date_time = time.ctime()\n",
    "                except:\n",
    "                    date_time = ''\n",
    "                try:\n",
    "                    detail_description_answer_all = div.find('section',class_='single_fatwa__question').find_next_sibling().find('section',class_='single_fatwa__answer').find('section',class_='single_fatwa__answer__body').div.find_all('p')\n",
    "                    detail_description_answer = ''\n",
    "                    for p in detail_description_answer_all:\n",
    "                        detail_description_answer = detail_description_answer + p.text\n",
    "                    detail_description = div.find('section',class_='single_fatwa__question').h2.text + ' ' + div.find('section',class_='single_fatwa__question').div.p.text + ' ' + div.find('section',class_='single_fatwa__question').find_next_sibling().find('section',class_='single_fatwa__answer').h2.text + ' ' + detail_description_answer \n",
    "                except:\n",
    "                    detail_description = ''\n",
    "                counter = counter + 1    \n",
    "                language='Urdu'\n",
    "                values = {\n",
    "                            'title': detail_title,\n",
    "                            'description': detail_description,\n",
    "                            'date': date_time,\n",
    "                            'channel': source[2],\n",
    "                            'image': high_resolution_image,\n",
    "                            'permalink': detail_page_source[0],\n",
    "                            'category': detail_page_source[1],\n",
    "                            'fetching_date': date.today(),\n",
    "                            'no_of_comments': 0,\n",
    "                            'rating': 0,\n",
    "                            'language':language,\n",
    "                            'created_at': date.today(),\n",
    "                            'updated_at': date.today(),\n",
    "                            'comments': [],\n",
    "                            'resgistered_views':[],\n",
    "                            'no_of_registered_views':0,\n",
    "                            'no_of_nonregistered_views':0,\n",
    "                            }\n",
    "                if developmentMode:\n",
    "                    data.append([detail_title, source[1], source[2], permalink, high_resolution_image, date_time, detail_description, date.today(), language])\n",
    "                    print('')\n",
    "                    print('No:          ' + str(counter))\n",
    "                    print('title:       ' + detail_title.strip())\n",
    "                    print('description: ' + detail_description[0:20])\n",
    "                    print('permalink:     ' + detail_page_source[0])\n",
    "                    print('channel:     ' + source[2])\n",
    "                    print('image:       ' + high_resolution_image)\n",
    "                    print('category:    ' + detail_page_source[1])\n",
    "                    print('-----------------------------------------------')\n",
    "                    if callAPI:\n",
    "                        response = requests.post(API, data=values)\n",
    "                        if response.status_code == 429:\n",
    "                            print('############ waiting response ..')\n",
    "                            time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "                        else:\n",
    "                            print('inserted in DB')\n",
    "                            print('-----------------------------------------------')\n",
    "                else:\n",
    "                    #data.append([detail_title.strip(), source[1], source[2], permalink, high_resolution_image, date_time, detail_description.strip(), date.today(), language])\n",
    "                    response = requests.post(API, data=values)\n",
    "                    if response.status_code == 429:\n",
    "                        time.sleep(int(response.headers[\"Retry-After\"]))\n",
    "        if developmentMode:\n",
    "            print('total news scraped page : ' + str(counter))\n",
    "except:\n",
    "    print('exception in ' + source[2])"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}