Web Scrap.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "r= requests.get('https://xkcd.com/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'<!DOCTYPE html>\\n<html>\\n<head>\\n<link rel=\"stylesheet\" type=\"text/css\" href=\"/s/b0dcca.css\" title=\"Default\"/>\\n<title>xkcd: When I\\'m Back at a Keyboard</title>\\n<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\"/>\\n<link rel=\"shortcut icon\" href=\"/s/919f27.ico\" type=\"image/x-icon\"/>\\n<link rel=\"icon\" href=\"/s/919f27.ico\" type=\"image/x-icon\"/>\\n<link rel=\"alternate\" type=\"application/atom+xml\" title=\"Atom 1.0\" href=\"/atom.xml\"/>\\n<link rel=\"alternate\" type=\"application/rss+xml\" title=\"RSS 2.0\" href=\"/rss.xml\"/>\\n<script type=\"text/javascript\" src=\"/s/b66ed7.js\" async></script>\\n<script type=\"text/javascript\" src=\"/s/1b9456.js\" async></script>\\n\\n<meta property=\"og:site_name\" content=\"xkcd\">\\n\\n<meta property=\"og:title\" content=\"When I&#39;m Back at a Keyboard\">\\n<meta property=\"og:url\" content=\"https://xkcd.com/2182/\">\\n<meta property=\"og:image\" content=\"https://imgs.xkcd.com/comics/when_im_back_at_a_keyboard_2x.png\">\\n<meta name=\"twitter:card\" content=\"summary_large_image\">\\n\\n</head>\\n<body>\\n<div id=\"topContainer\">\\n<div id=\"topLeft\">\\n<ul>\\n<li><a href=\"/archive\">Archive</a></li>\\n<li><a href=\"http://what-if.xkcd.com\">What If?</a></li>\\n<li><a href=\"http://blag.xkcd.com\">Blag</a></li>\\n<li><a href=\"/how-to/\">How To</a></li>\\n<li><a href=\"http://store.xkcd.com/\">Store</a></li>\\n<li><a rel=\"author\" href=\"/about\">About</a></li>\\n<li><a href=\"/atom.xml\">Feed</a> &bull; <a href=\"/newsletter/\">Email</a></li>\\n</ul>\\n</div>\\n<div id=\"topRight\">\\n<div id=\"masthead\">\\n<span><a href=\"/\"><img src=\"/s/0b7742.png\" alt=\"xkcd.com logo\" height=\"83\" width=\"185\"/></a></span>\\n<span id=\"slogan\">A webcomic of romance,<br/> sarcasm, math, and language.</span>\\n</div>\\n<div id=\"news\">\\n<div id=\"htNews\">\\n<a href=\"//xkcd.com/how-to/\"><img border=0 src=\"//imgs.xkcd.com/static/how_to_news.png\"></a>\\n  Preorder:\\n    <a href=\"https://www.amazon.com/How-Absurd-Scientific-Real-World-Problems/dp/0525537090/\">Amazon</a>,\\n    <a href=\"http://links.penguinrandomhouse.com/type/affiliate/isbn/9780525537090/siteID/8001/retailerid/2/trackingcode/PRH5522E62429\">Barnes & Noble</a>,\\n    <a href=\"http://links.penguinrandomhouse.com/type/affiliate/isbn/9780525537090/siteID/8001/retailerid/6/trackingcode/penguinrandom\">IndieBound</a>,\\n    <a href=\"https://itunes.apple.com/us/book/how-to/id1451461524?mt=11\">Apple Books</a>\\n</div>\\n<script>\\nvar client = new XMLHttpRequest();\\nclient.open(\"GET\", \"//c.xkcd.com/how-to/news\", true);\\nclient.send();\\nclient.onreadystatechange = function() {\\n  if(client.readyState == 4 && client.status == 200) {\\n    document.getElementById(\"htNews\").innerHTML = client.responseText;\\n  }\\n}\\n</script>\\n\\n</div>\\n</div>\\n<div id=\"bgLeft\" class=\"bg box\"></div>\\n<div id=\"bgRight\" class=\"bg box\"></div>\\n</div>\\n<div id=\"middleContainer\" class=\"box\">\\n\\n<div id=\"ctitle\">When I\\'m Back at a Keyboard</div>\\n<ul class=\"comicNav\">\\n<li><a href=\"/1/\">|&lt;</a></li>\\n<li><a rel=\"prev\" href=\"/2181/\" accesskey=\"p\">&lt; Prev</a></li>\\n<li><a href=\"//c.xkcd.com/random/comic/\">Random</a></li>\\n<li><a rel=\"next\" href=\"#\" accesskey=\"n\">Next &gt;</a></li>\\n<li><a href=\"/\">&gt;|</a></li>\\n</ul>\\n<div id=\"comic\">\\n<img src=\"//imgs.xkcd.com/comics/when_im_back_at_a_keyboard.png\" title=\"[after typing 1,500 words on feathered dinosaurs, paleontology, sexism, lava, and dinosaurs as animals rather than movie monsters] Sorry to cut it short, I&#39;m on my phone. When I&#39;m back at a keyboard, I can give you another 5,000 words.\" alt=\"When I&#39;m Back at a Keyboard\" srcset=\"//imgs.xkcd.com/comics/when_im_back_at_a_keyboard_2x.png 2x\"/>\\n</div>\\n<ul class=\"comicNav\">\\n<li><a href=\"/1/\">|&lt;</a></li>\\n<li><a rel=\"prev\" href=\"/2181/\" accesskey=\"p\">&lt; Prev</a></li>\\n<li><a href=\"//c.xkcd.com/random/comic/\">Random</a></li>\\n<li><a rel=\"next\" href=\"#\" accesskey=\"n\">Next &gt;</a></li>\\n<li><a href=\"/\">&gt;|</a></li>\\n</ul>\\n<br />\\nPermanent link to this comic: https://xkcd.com/2182/<br />\\nImage URL (for hotlinking/embedding): https://imgs.xkcd.com/comics/when_im_back_at_a_keyboard.png\\n<div id=\"transcript\" style=\"display: none\"></div>\\n</div>\\n<div id=\"bottom\" class=\"box\">\\n<img src=\"//imgs.xkcd.com/s/a899e84.jpg\" width=\"520\" height=\"100\" alt=\"Selected Comics\" usemap=\"#comicmap\"/>\\n<map id=\"comicmap\" name=\"comicmap\">\\n<area shape=\"rect\" coords=\"0,0,100,100\" href=\"/150/\" alt=\"Grownups\"/>\\n<area shape=\"rect\" coords=\"104,0,204,100\" href=\"/730/\" alt=\"Circuit Diagram\"/>\\n<area shape=\"rect\" coords=\"208,0,308,100\" href=\"/162/\" alt=\"Angular Momentum\"/>\\n<area shape=\"rect\" coords=\"312,0,412,100\" href=\"/688/\" alt=\"Self-Description\"/>\\n<area shape=\"rect\" coords=\"416,0,520,100\" href=\"/556/\" alt=\"Alternative Energy Revolution\"/>\\n</map>\\n<br />\\n<a href=\"//xkcd.com/1732/\"><img border=0 src=\"//imgs.xkcd.com/s/temperature.png\" width=\"520\" height=\"100\" alt=\"Earth temperature timeline\"></a>\\n<br />\\n<div>\\n<!--\\nSearch comic titles and transcripts:\\n<script type=\"text/javascript\" src=\"//www.google.com/jsapi\"></script>\\n<script type=\"text/javascript\">google.load(\\'search\\', \\'1\\');google.setOnLoadCallback(function() {google.search.CustomSearchControl.attachAutoCompletion(\\'012652707207066138651:zudjtuwe28q\\',document.getElementById(\\'q\\'),\\'cse-search-box\\');});</script>\\n<form action=\"//www.google.com/cse\" id=\"cse-search-box\">\\n<div>\\n<input type=\"hidden\" name=\"cx\" value=\"012652707207066138651:zudjtuwe28q\"/>\\n<input type=\"hidden\" name=\"ie\" value=\"UTF-8\"/>\\n<input type=\"text\" name=\"q\" id=\"q\" size=\"31\"/>\\n<input type=\"submit\" name=\"sa\" value=\"Search\"/>\\n</div>\\n</form>\\n<script type=\"text/javascript\" src=\"//www.google.com/cse/brand?form=cse-search-box&amp;lang=en\"></script>\\n-->\\n<a href=\"/rss.xml\">RSS Feed</a> - <a href=\"/atom.xml\">Atom Feed</a> - <a href=\"/newsletter/\">Email</a>\\n</div>\\n<br />\\n<div id=\"comicLinks\">\\nComics I enjoy:<br/>\\n        <a href=\"http://threewordphrase.com/\">Three Word Phrase</a>,\\n        <a href=\"http://www.smbc-comics.com/\">SMBC</a>,\\n        <a href=\"http://www.qwantz.com\">Dinosaur Comics</a>,\\n        <a href=\"http://oglaf.com/\">Oglaf</a> (nsfw),\\n        <a href=\"http://www.asofterworld.com\">A Softer World</a>,\\n        <a href=\"http://buttersafe.com/\">Buttersafe</a>,\\n        <a href=\"http://pbfcomics.com/\">Perry Bible Fellowship</a>,\\n        <a href=\"http://questionablecontent.net/\">Questionable Content</a>,\\n        <a href=\"http://www.buttercupfestival.com/\">Buttercup Festival</a>,\\n        <a href=\"http://www.mspaintadventures.com/?s=6&p=001901\">Homestuck</a>,\\n\\t<a href=\"http://www.jspowerhour.com/\">Junior Scientist Power Hour</a>\\n</div>\\n<br />\\n<div id=\"comicLinks\">\\nOther things:<br/>\\n        <a href=\"https://medium.com/civic-tech-thoughts-from-joshdata/so-you-want-to-reform-democracy-7f3b1ef10597\">Tips on technology and government</a>,<br /> \\n        <a href=\"https://www.nytimes.com/interactive/2017/climate/what-is-climate-change.html\">Climate FAQ</a>,\\n\\t<a href=\"https://twitter.com/KHayhoe\">Katharine Hayhoe</a>\\n</div>\\n<br />\\n<center>\\n<div id=\"footnote\" style=\"width:70%\">xkcd.com is best viewed with Netscape Navigator 4.0 or below on a Pentium 3&plusmn;1 emulated in Javascript on an Apple IIGS<br />at a screen resolution of 1024x1. Please enable your ad blockers, disable high-heat drying, and remove your device<br />from Airplane Mode and set it to Boat Mode. For security reasons, please leave caps lock on while browsing.</div>\\n</center>\\n<div id=\"licenseText\">\\n<p>\\nThis work is licensed under a\\n<a href=\"http://creativecommons.org/licenses/by-nc/2.5/\">Creative Commons Attribution-NonCommercial 2.5 License</a>.\\n</p><p>\\nThis means you\\'re free to copy and share these comics (but not to sell them). <a rel=\"license\" href=\"/license.html\">More details</a>.</p>\\n</div>\\n</div>\\n</body>\\n<!-- Layout by Ian Clasbey, davean, and chromakode -->\\n</html>\\n\\n'\n"
     ]
    }
   ],
   "source": [
    "print(r.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup= bs(r.content, 'html5lib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html><head>\n",
      "<link href=\"/s/b0dcca.css\" rel=\"stylesheet\" title=\"Default\" type=\"text/css\"/>\n",
      "<title>xkcd: When I'm Back at a Keyboard</title>\n",
      "<meta content=\"IE=edge\" http-equiv=\"X-UA-Compatible\"/>\n",
      "<link href=\"/s/919f27.ico\" rel=\"shortcut icon\" type=\"image/x-icon\"/>\n",
      "<link href=\"/s/919f27.ico\" rel=\"icon\" type=\"image/x-icon\"/>\n",
      "<link href=\"/atom.xml\" rel=\"alternate\" title=\"Atom 1.0\" type=\"application/atom+xml\"/>\n",
      "<link href=\"/rss.xml\" rel=\"alternate\" title=\"RSS 2.0\" type=\"application/rss+xml\"/>\n",
      "<script async=\"\" src=\"/s/b66ed7.js\" type=\"text/javascript\"></script>\n",
      "<script async=\"\" src=\"/s/1b9456.js\" type=\"text/javascript\"></script>\n",
      "\n",
      "<meta content=\"xkcd\" property=\"og:site_name\"/>\n",
      "\n",
      "<meta content=\"When I'm Back at a Keyboard\" property=\"og:title\"/>\n",
      "<meta content=\"https://xkcd.com/2182/\" property=\"og:url\"/>\n",
      "<meta content=\"https://imgs.xkcd.com/comics/when_im_back_at_a_keyboard_2x.png\" property=\"og:image\"/>\n",
      "<meta content=\"summary_large_image\" name=\"twitter:card\"/>\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"topContainer\">\n",
      "<div id=\"topLeft\">\n",
      "<ul>\n",
      "<li><a href=\"/archive\">Archive</a></li>\n",
      "<li><a href=\"http://what-if.xkcd.com\">What If?</a></li>\n",
      "<li><a href=\"http://blag.xkcd.com\">Blag</a></li>\n",
      "<li><a href=\"/how-to/\">How To</a></li>\n",
      "<li><a href=\"http://store.xkcd.com/\">Store</a></li>\n",
      "<li><a href=\"/about\" rel=\"author\">About</a></li>\n",
      "<li><a href=\"/atom.xml\">Feed</a> • <a href=\"/newsletter/\">Email</a></li>\n",
      "</ul>\n",
      "</div>\n",
      "<div id=\"topRight\">\n",
      "<div id=\"masthead\">\n",
      "<span><a href=\"/\"><img alt=\"xkcd.com logo\" height=\"83\" src=\"/s/0b7742.png\" width=\"185\"/></a></span>\n",
      "<span id=\"slogan\">A webcomic of romance,<br/> sarcasm, math, and language.</span>\n",
      "</div>\n",
      "<div id=\"news\">\n",
      "<div id=\"htNews\">\n",
      "<a href=\"//xkcd.com/how-to/\"><img border=\"0\" src=\"//imgs.xkcd.com/static/how_to_news.png\"/></a>\n",
      "  Preorder:\n",
      "    <a href=\"https://www.amazon.com/How-Absurd-Scientific-Real-World-Problems/dp/0525537090/\">Amazon</a>,\n",
      "    <a href=\"http://links.penguinrandomhouse.com/type/affiliate/isbn/9780525537090/siteID/8001/retailerid/2/trackingcode/PRH5522E62429\">Barnes &amp; Noble</a>,\n",
      "    <a href=\"http://links.penguinrandomhouse.com/type/affiliate/isbn/9780525537090/siteID/8001/retailerid/6/trackingcode/penguinrandom\">IndieBound</a>,\n",
      "    <a href=\"https://itunes.apple.com/us/book/how-to/id1451461524?mt=11\">Apple Books</a>\n",
      "</div>\n",
      "<script>\n",
      "var client = new XMLHttpRequest();\n",
      "client.open(\"GET\", \"//c.xkcd.com/how-to/news\", true);\n",
      "client.send();\n",
      "client.onreadystatechange = function() {\n",
      "  if(client.readyState == 4 && client.status == 200) {\n",
      "    document.getElementById(\"htNews\").innerHTML = client.responseText;\n",
      "  }\n",
      "}\n",
      "</script>\n",
      "\n",
      "</div>\n",
      "</div>\n",
      "<div class=\"bg box\" id=\"bgLeft\"></div>\n",
      "<div class=\"bg box\" id=\"bgRight\"></div>\n",
      "</div>\n",
      "<div class=\"box\" id=\"middleContainer\">\n",
      "\n",
      "<div id=\"ctitle\">When I'm Back at a Keyboard</div>\n",
      "<ul class=\"comicNav\">\n",
      "<li><a href=\"/1/\">|&lt;</a></li>\n",
      "<li><a accesskey=\"p\" href=\"/2181/\" rel=\"prev\">&lt; Prev</a></li>\n",
      "<li><a href=\"//c.xkcd.com/random/comic/\">Random</a></li>\n",
      "<li><a accesskey=\"n\" href=\"#\" rel=\"next\">Next &gt;</a></li>\n",
      "<li><a href=\"/\">&gt;|</a></li>\n",
      "</ul>\n",
      "<div id=\"comic\">\n",
      "<img alt=\"When I'm Back at a Keyboard\" src=\"//imgs.xkcd.com/comics/when_im_back_at_a_keyboard.png\" srcset=\"//imgs.xkcd.com/comics/when_im_back_at_a_keyboard_2x.png 2x\" title=\"[after typing 1,500 words on feathered dinosaurs, paleontology, sexism, lava, and dinosaurs as animals rather than movie monsters] Sorry to cut it short, I'm on my phone. When I'm back at a keyboard, I can give you another 5,000 words.\"/>\n",
      "</div>\n",
      "<ul class=\"comicNav\">\n",
      "<li><a href=\"/1/\">|&lt;</a></li>\n",
      "<li><a accesskey=\"p\" href=\"/2181/\" rel=\"prev\">&lt; Prev</a></li>\n",
      "<li><a href=\"//c.xkcd.com/random/comic/\">Random</a></li>\n",
      "<li><a accesskey=\"n\" href=\"#\" rel=\"next\">Next &gt;</a></li>\n",
      "<li><a href=\"/\">&gt;|</a></li>\n",
      "</ul>\n",
      "<br/>\n",
      "Permanent link to this comic: https://xkcd.com/2182/<br/>\n",
      "Image URL (for hotlinking/embedding): https://imgs.xkcd.com/comics/when_im_back_at_a_keyboard.png\n",
      "<div id=\"transcript\" style=\"display: none\"></div>\n",
      "</div>\n",
      "<div class=\"box\" id=\"bottom\">\n",
      "<img alt=\"Selected Comics\" height=\"100\" src=\"//imgs.xkcd.com/s/a899e84.jpg\" usemap=\"#comicmap\" width=\"520\"/>\n",
      "<map id=\"comicmap\" name=\"comicmap\">\n",
      "<area alt=\"Grownups\" coords=\"0,0,100,100\" href=\"/150/\" shape=\"rect\"/>\n",
      "<area alt=\"Circuit Diagram\" coords=\"104,0,204,100\" href=\"/730/\" shape=\"rect\"/>\n",
      "<area alt=\"Angular Momentum\" coords=\"208,0,308,100\" href=\"/162/\" shape=\"rect\"/>\n",
      "<area alt=\"Self-Description\" coords=\"312,0,412,100\" href=\"/688/\" shape=\"rect\"/>\n",
      "<area alt=\"Alternative Energy Revolution\" coords=\"416,0,520,100\" href=\"/556/\" shape=\"rect\"/>\n",
      "</map>\n",
      "<br/>\n",
      "<a href=\"//xkcd.com/1732/\"><img alt=\"Earth temperature timeline\" border=\"0\" height=\"100\" src=\"//imgs.xkcd.com/s/temperature.png\" width=\"520\"/></a>\n",
      "<br/>\n",
      "<div>\n",
      "<!--\n",
      "Search comic titles and transcripts:\n",
      "<script type=\"text/javascript\" src=\"//www.google.com/jsapi\"></script>\n",
      "<script type=\"text/javascript\">google.load('search', '1');google.setOnLoadCallback(function() {google.search.CustomSearchControl.attachAutoCompletion('012652707207066138651:zudjtuwe28q',document.getElementById('q'),'cse-search-box');});</script>\n",
      "<form action=\"//www.google.com/cse\" id=\"cse-search-box\">\n",
      "<div>\n",
      "<input type=\"hidden\" name=\"cx\" value=\"012652707207066138651:zudjtuwe28q\"/>\n",
      "<input type=\"hidden\" name=\"ie\" value=\"UTF-8\"/>\n",
      "<input type=\"text\" name=\"q\" id=\"q\" size=\"31\"/>\n",
      "<input type=\"submit\" name=\"sa\" value=\"Search\"/>\n",
      "</div>\n",
      "</form>\n",
      "<script type=\"text/javascript\" src=\"//www.google.com/cse/brand?form=cse-search-box&amp;lang=en\"></script>\n",
      "-->\n",
      "<a href=\"/rss.xml\">RSS Feed</a> - <a href=\"/atom.xml\">Atom Feed</a> - <a href=\"/newsletter/\">Email</a>\n",
      "</div>\n",
      "<br/>\n",
      "<div id=\"comicLinks\">\n",
      "Comics I enjoy:<br/>\n",
      "        <a href=\"http://threewordphrase.com/\">Three Word Phrase</a>,\n",
      "        <a href=\"http://www.smbc-comics.com/\">SMBC</a>,\n",
      "        <a href=\"http://www.qwantz.com\">Dinosaur Comics</a>,\n",
      "        <a href=\"http://oglaf.com/\">Oglaf</a> (nsfw),\n",
      "        <a href=\"http://www.asofterworld.com\">A Softer World</a>,\n",
      "        <a href=\"http://buttersafe.com/\">Buttersafe</a>,\n",
      "        <a href=\"http://pbfcomics.com/\">Perry Bible Fellowship</a>,\n",
      "        <a href=\"http://questionablecontent.net/\">Questionable Content</a>,\n",
      "        <a href=\"http://www.buttercupfestival.com/\">Buttercup Festival</a>,\n",
      "        <a href=\"http://www.mspaintadventures.com/?s=6&amp;p=001901\">Homestuck</a>,\n",
      "\t<a href=\"http://www.jspowerhour.com/\">Junior Scientist Power Hour</a>\n",
      "</div>\n",
      "<br/>\n",
      "<div id=\"comicLinks\">\n",
      "Other things:<br/>\n",
      "        <a href=\"https://medium.com/civic-tech-thoughts-from-joshdata/so-you-want-to-reform-democracy-7f3b1ef10597\">Tips on technology and government</a>,<br/> \n",
      "        <a href=\"https://www.nytimes.com/interactive/2017/climate/what-is-climate-change.html\">Climate FAQ</a>,\n",
      "\t<a href=\"https://twitter.com/KHayhoe\">Katharine Hayhoe</a>\n",
      "</div>\n",
      "<br/>\n",
      "<center>\n",
      "<div id=\"footnote\" style=\"width:70%\">xkcd.com is best viewed with Netscape Navigator 4.0 or below on a Pentium 3±1 emulated in Javascript on an Apple IIGS<br/>at a screen resolution of 1024x1. Please enable your ad blockers, disable high-heat drying, and remove your device<br/>from Airplane Mode and set it to Boat Mode. For security reasons, please leave caps lock on while browsing.</div>\n",
      "</center>\n",
      "<div id=\"licenseText\">\n",
      "<p>\n",
      "This work is licensed under a\n",
      "<a href=\"http://creativecommons.org/licenses/by-nc/2.5/\">Creative Commons Attribution-NonCommercial 2.5 License</a>.\n",
      "</p><p>\n",
      "This means you're free to copy and share these comics (but not to sell them). <a href=\"/license.html\" rel=\"license\">More details</a>.</p>\n",
      "</div>\n",
      "</div>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "</body><!-- Layout by Ian Clasbey, davean, and chromakode --></html>\n"
     ]
    }
   ],
   "source": [
    "print(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method Tag.prettify of <!DOCTYPE html>\n",
      "<html><head>\n",
      "<link href=\"/s/b0dcca.css\" rel=\"stylesheet\" title=\"Default\" type=\"text/css\"/>\n",
      "<title>xkcd: When I'm Back at a Keyboard</title>\n",
      "<meta content=\"IE=edge\" http-equiv=\"X-UA-Compatible\"/>\n",
      "<link href=\"/s/919f27.ico\" rel=\"shortcut icon\" type=\"image/x-icon\"/>\n",
      "<link href=\"/s/919f27.ico\" rel=\"icon\" type=\"image/x-icon\"/>\n",
      "<link href=\"/atom.xml\" rel=\"alternate\" title=\"Atom 1.0\" type=\"application/atom+xml\"/>\n",
      "<link href=\"/rss.xml\" rel=\"alternate\" title=\"RSS 2.0\" type=\"application/rss+xml\"/>\n",
      "<script async=\"\" src=\"/s/b66ed7.js\" type=\"text/javascript\"></script>\n",
      "<script async=\"\" src=\"/s/1b9456.js\" type=\"text/javascript\"></script>\n",
      "\n",
      "<meta content=\"xkcd\" property=\"og:site_name\"/>\n",
      "\n",
      "<meta content=\"When I'm Back at a Keyboard\" property=\"og:title\"/>\n",
      "<meta content=\"https://xkcd.com/2182/\" property=\"og:url\"/>\n",
      "<meta content=\"https://imgs.xkcd.com/comics/when_im_back_at_a_keyboard_2x.png\" property=\"og:image\"/>\n",
      "<meta content=\"summary_large_image\" name=\"twitter:card\"/>\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"topContainer\">\n",
      "<div id=\"topLeft\">\n",
      "<ul>\n",
      "<li><a href=\"/archive\">Archive</a></li>\n",
      "<li><a href=\"http://what-if.xkcd.com\">What If?</a></li>\n",
      "<li><a href=\"http://blag.xkcd.com\">Blag</a></li>\n",
      "<li><a href=\"/how-to/\">How To</a></li>\n",
      "<li><a href=\"http://store.xkcd.com/\">Store</a></li>\n",
      "<li><a href=\"/about\" rel=\"author\">About</a></li>\n",
      "<li><a href=\"/atom.xml\">Feed</a> • <a href=\"/newsletter/\">Email</a></li>\n",
      "</ul>\n",
      "</div>\n",
      "<div id=\"topRight\">\n",
      "<div id=\"masthead\">\n",
      "<span><a href=\"/\"><img alt=\"xkcd.com logo\" height=\"83\" src=\"/s/0b7742.png\" width=\"185\"/></a></span>\n",
      "<span id=\"slogan\">A webcomic of romance,<br/> sarcasm, math, and language.</span>\n",
      "</div>\n",
      "<div id=\"news\">\n",
      "<div id=\"htNews\">\n",
      "<a href=\"//xkcd.com/how-to/\"><img border=\"0\" src=\"//imgs.xkcd.com/static/how_to_news.png\"/></a>\n",
      "  Preorder:\n",
      "    <a href=\"https://www.amazon.com/How-Absurd-Scientific-Real-World-Problems/dp/0525537090/\">Amazon</a>,\n",
      "    <a href=\"http://links.penguinrandomhouse.com/type/affiliate/isbn/9780525537090/siteID/8001/retailerid/2/trackingcode/PRH5522E62429\">Barnes &amp; Noble</a>,\n",
      "    <a href=\"http://links.penguinrandomhouse.com/type/affiliate/isbn/9780525537090/siteID/8001/retailerid/6/trackingcode/penguinrandom\">IndieBound</a>,\n",
      "    <a href=\"https://itunes.apple.com/us/book/how-to/id1451461524?mt=11\">Apple Books</a>\n",
      "</div>\n",
      "<script>\n",
      "var client = new XMLHttpRequest();\n",
      "client.open(\"GET\", \"//c.xkcd.com/how-to/news\", true);\n",
      "client.send();\n",
      "client.onreadystatechange = function() {\n",
      "  if(client.readyState == 4 && client.status == 200) {\n",
      "    document.getElementById(\"htNews\").innerHTML = client.responseText;\n",
      "  }\n",
      "}\n",
      "</script>\n",
      "\n",
      "</div>\n",
      "</div>\n",
      "<div class=\"bg box\" id=\"bgLeft\"></div>\n",
      "<div class=\"bg box\" id=\"bgRight\"></div>\n",
      "</div>\n",
      "<div class=\"box\" id=\"middleContainer\">\n",
      "\n",
      "<div id=\"ctitle\">When I'm Back at a Keyboard</div>\n",
      "<ul class=\"comicNav\">\n",
      "<li><a href=\"/1/\">|&lt;</a></li>\n",
      "<li><a accesskey=\"p\" href=\"/2181/\" rel=\"prev\">&lt; Prev</a></li>\n",
      "<li><a href=\"//c.xkcd.com/random/comic/\">Random</a></li>\n",
      "<li><a accesskey=\"n\" href=\"#\" rel=\"next\">Next &gt;</a></li>\n",
      "<li><a href=\"/\">&gt;|</a></li>\n",
      "</ul>\n",
      "<div id=\"comic\">\n",
      "<img alt=\"When I'm Back at a Keyboard\" src=\"//imgs.xkcd.com/comics/when_im_back_at_a_keyboard.png\" srcset=\"//imgs.xkcd.com/comics/when_im_back_at_a_keyboard_2x.png 2x\" title=\"[after typing 1,500 words on feathered dinosaurs, paleontology, sexism, lava, and dinosaurs as animals rather than movie monsters] Sorry to cut it short, I'm on my phone. When I'm back at a keyboard, I can give you another 5,000 words.\"/>\n",
      "</div>\n",
      "<ul class=\"comicNav\">\n",
      "<li><a href=\"/1/\">|&lt;</a></li>\n",
      "<li><a accesskey=\"p\" href=\"/2181/\" rel=\"prev\">&lt; Prev</a></li>\n",
      "<li><a href=\"//c.xkcd.com/random/comic/\">Random</a></li>\n",
      "<li><a accesskey=\"n\" href=\"#\" rel=\"next\">Next &gt;</a></li>\n",
      "<li><a href=\"/\">&gt;|</a></li>\n",
      "</ul>\n",
      "<br/>\n",
      "Permanent link to this comic: https://xkcd.com/2182/<br/>\n",
      "Image URL (for hotlinking/embedding): https://imgs.xkcd.com/comics/when_im_back_at_a_keyboard.png\n",
      "<div id=\"transcript\" style=\"display: none\"></div>\n",
      "</div>\n",
      "<div class=\"box\" id=\"bottom\">\n",
      "<img alt=\"Selected Comics\" height=\"100\" src=\"//imgs.xkcd.com/s/a899e84.jpg\" usemap=\"#comicmap\" width=\"520\"/>\n",
      "<map id=\"comicmap\" name=\"comicmap\">\n",
      "<area alt=\"Grownups\" coords=\"0,0,100,100\" href=\"/150/\" shape=\"rect\"/>\n",
      "<area alt=\"Circuit Diagram\" coords=\"104,0,204,100\" href=\"/730/\" shape=\"rect\"/>\n",
      "<area alt=\"Angular Momentum\" coords=\"208,0,308,100\" href=\"/162/\" shape=\"rect\"/>\n",
      "<area alt=\"Self-Description\" coords=\"312,0,412,100\" href=\"/688/\" shape=\"rect\"/>\n",
      "<area alt=\"Alternative Energy Revolution\" coords=\"416,0,520,100\" href=\"/556/\" shape=\"rect\"/>\n",
      "</map>\n",
      "<br/>\n",
      "<a href=\"//xkcd.com/1732/\"><img alt=\"Earth temperature timeline\" border=\"0\" height=\"100\" src=\"//imgs.xkcd.com/s/temperature.png\" width=\"520\"/></a>\n",
      "<br/>\n",
      "<div>\n",
      "<!--\n",
      "Search comic titles and transcripts:\n",
      "<script type=\"text/javascript\" src=\"//www.google.com/jsapi\"></script>\n",
      "<script type=\"text/javascript\">google.load('search', '1');google.setOnLoadCallback(function() {google.search.CustomSearchControl.attachAutoCompletion('012652707207066138651:zudjtuwe28q',document.getElementById('q'),'cse-search-box');});</script>\n",
      "<form action=\"//www.google.com/cse\" id=\"cse-search-box\">\n",
      "<div>\n",
      "<input type=\"hidden\" name=\"cx\" value=\"012652707207066138651:zudjtuwe28q\"/>\n",
      "<input type=\"hidden\" name=\"ie\" value=\"UTF-8\"/>\n",
      "<input type=\"text\" name=\"q\" id=\"q\" size=\"31\"/>\n",
      "<input type=\"submit\" name=\"sa\" value=\"Search\"/>\n",
      "</div>\n",
      "</form>\n",
      "<script type=\"text/javascript\" src=\"//www.google.com/cse/brand?form=cse-search-box&amp;lang=en\"></script>\n",
      "-->\n",
      "<a href=\"/rss.xml\">RSS Feed</a> - <a href=\"/atom.xml\">Atom Feed</a> - <a href=\"/newsletter/\">Email</a>\n",
      "</div>\n",
      "<br/>\n",
      "<div id=\"comicLinks\">\n",
      "Comics I enjoy:<br/>\n",
      "        <a href=\"http://threewordphrase.com/\">Three Word Phrase</a>,\n",
      "        <a href=\"http://www.smbc-comics.com/\">SMBC</a>,\n",
      "        <a href=\"http://www.qwantz.com\">Dinosaur Comics</a>,\n",
      "        <a href=\"http://oglaf.com/\">Oglaf</a> (nsfw),\n",
      "        <a href=\"http://www.asofterworld.com\">A Softer World</a>,\n",
      "        <a href=\"http://buttersafe.com/\">Buttersafe</a>,\n",
      "        <a href=\"http://pbfcomics.com/\">Perry Bible Fellowship</a>,\n",
      "        <a href=\"http://questionablecontent.net/\">Questionable Content</a>,\n",
      "        <a href=\"http://www.buttercupfestival.com/\">Buttercup Festival</a>,\n",
      "        <a href=\"http://www.mspaintadventures.com/?s=6&amp;p=001901\">Homestuck</a>,\n",
      "\t<a href=\"http://www.jspowerhour.com/\">Junior Scientist Power Hour</a>\n",
      "</div>\n",
      "<br/>\n",
      "<div id=\"comicLinks\">\n",
      "Other things:<br/>\n",
      "        <a href=\"https://medium.com/civic-tech-thoughts-from-joshdata/so-you-want-to-reform-democracy-7f3b1ef10597\">Tips on technology and government</a>,<br/> \n",
      "        <a href=\"https://www.nytimes.com/interactive/2017/climate/what-is-climate-change.html\">Climate FAQ</a>,\n",
      "\t<a href=\"https://twitter.com/KHayhoe\">Katharine Hayhoe</a>\n",
      "</div>\n",
      "<br/>\n",
      "<center>\n",
      "<div id=\"footnote\" style=\"width:70%\">xkcd.com is best viewed with Netscape Navigator 4.0 or below on a Pentium 3±1 emulated in Javascript on an Apple IIGS<br/>at a screen resolution of 1024x1. Please enable your ad blockers, disable high-heat drying, and remove your device<br/>from Airplane Mode and set it to Boat Mode. For security reasons, please leave caps lock on while browsing.</div>\n",
      "</center>\n",
      "<div id=\"licenseText\">\n",
      "<p>\n",
      "This work is licensed under a\n",
      "<a href=\"http://creativecommons.org/licenses/by-nc/2.5/\">Creative Commons Attribution-NonCommercial 2.5 License</a>.\n",
      "</p><p>\n",
      "This means you're free to copy and share these comics (but not to sell them). <a href=\"/license.html\" rel=\"license\">More details</a>.</p>\n",
      "</div>\n",
      "</div>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "</body><!-- Layout by Ian Clasbey, davean, and chromakode --></html>>\n"
     ]
    }
   ],
   "source": [
    "print(soup.prettify)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=soup.find(id='comic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div id=\"comic\">\n",
      "<img alt=\"When I'm Back at a Keyboard\" src=\"//imgs.xkcd.com/comics/when_im_back_at_a_keyboard.png\" srcset=\"//imgs.xkcd.com/comics/when_im_back_at_a_keyboard_2x.png 2x\" title=\"[after typing 1,500 words on feathered dinosaurs, paleontology, sexism, lava, and dinosaurs as animals rather than movie monsters] Sorry to cut it short, I'm on my phone. When I'm back at a keyboard, I can give you another 5,000 words.\"/>\n",
      "</div>\n"
     ]
    }
   ],
   "source": [
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "    import urllib.request\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\New folder\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file F:\\New folder\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<img alt=\"xkcd.com logo\" height=\"83\" src=\"/s/0b7742.png\" width=\"185\"/>, <img border=\"0\" src=\"//imgs.xkcd.com/static/how_to_news.png\"/>, <img alt=\"Barrel - Part 1\" src=\"//imgs.xkcd.com/comics/barrel_cropped_(1).jpg\" title=\"Don't we all.\"/>, <img alt=\"Selected Comics\" height=\"100\" src=\"//imgs.xkcd.com/s/a899e84.jpg\" usemap=\"#comicmap\" width=\"520\"/>, <img alt=\"Earth temperature timeline\" border=\"0\" height=\"100\" src=\"//imgs.xkcd.com/s/temperature.png\" width=\"520\"/>]\n",
      "/s/0b7742.png\n",
      "0\n",
      "https:/s/0b7742.png\n",
      "<urlopen error no host given>\n",
      "//imgs.xkcd.com/static/how_to_news.png\n",
      "1\n",
      "https://imgs.xkcd.com/static/how_to_news.png\n",
      "//imgs.xkcd.com/comics/barrel_cropped_(1).jpg\n",
      "2\n",
      "https://imgs.xkcd.com/comics/barrel_cropped_(1).jpg\n",
      "//imgs.xkcd.com/s/a899e84.jpg\n",
      "3\n",
      "https://imgs.xkcd.com/s/a899e84.jpg\n",
      "//imgs.xkcd.com/s/temperature.png\n",
      "4\n",
      "https://imgs.xkcd.com/s/temperature.png\n",
      "2\n",
      "[<img alt=\"xkcd.com logo\" height=\"83\" src=\"/s/0b7742.png\" width=\"185\"/>, <img border=\"0\" src=\"//imgs.xkcd.com/static/how_to_news.png\"/>, <img alt=\"Petit Trees (sketch)\" src=\"//imgs.xkcd.com/comics/tree_cropped_(1).jpg\" title=\"'Petit' being a reference to Le Petit Prince, which I only thought about halfway through the sketch\"/>, <img alt=\"Selected Comics\" height=\"100\" src=\"//imgs.xkcd.com/s/a899e84.jpg\" usemap=\"#comicmap\" width=\"520\"/>, <img alt=\"Earth temperature timeline\" border=\"0\" height=\"100\" src=\"//imgs.xkcd.com/s/temperature.png\" width=\"520\"/>]\n",
      "/s/0b7742.png\n",
      "0\n",
      "https:/s/0b7742.png\n",
      "<urlopen error no host given>\n",
      "//imgs.xkcd.com/static/how_to_news.png\n",
      "1\n",
      "https://imgs.xkcd.com/static/how_to_news.png\n",
      "//imgs.xkcd.com/comics/tree_cropped_(1).jpg\n",
      "2\n",
      "https://imgs.xkcd.com/comics/tree_cropped_(1).jpg\n",
      "//imgs.xkcd.com/s/a899e84.jpg\n",
      "3\n",
      "https://imgs.xkcd.com/s/a899e84.jpg\n",
      "//imgs.xkcd.com/s/temperature.png\n",
      "4\n",
      "https://imgs.xkcd.com/s/temperature.png\n",
      "3\n",
      "[<img alt=\"xkcd.com logo\" height=\"83\" src=\"/s/0b7742.png\" width=\"185\"/>, <img border=\"0\" src=\"//imgs.xkcd.com/static/how_to_news.png\"/>, <img alt=\"Island (sketch)\" src=\"//imgs.xkcd.com/comics/island_color.jpg\" title=\"Hello, island\"/>, <img alt=\"Selected Comics\" height=\"100\" src=\"//imgs.xkcd.com/s/a899e84.jpg\" usemap=\"#comicmap\" width=\"520\"/>, <img alt=\"Earth temperature timeline\" border=\"0\" height=\"100\" src=\"//imgs.xkcd.com/s/temperature.png\" width=\"520\"/>]\n",
      "/s/0b7742.png\n",
      "0\n",
      "https:/s/0b7742.png\n",
      "<urlopen error no host given>\n",
      "//imgs.xkcd.com/static/how_to_news.png\n",
      "1\n",
      "https://imgs.xkcd.com/static/how_to_news.png\n",
      "//imgs.xkcd.com/comics/island_color.jpg\n",
      "2\n",
      "https://imgs.xkcd.com/comics/island_color.jpg\n",
      "//imgs.xkcd.com/s/a899e84.jpg\n",
      "3\n",
      "https://imgs.xkcd.com/s/a899e84.jpg\n",
      "//imgs.xkcd.com/s/temperature.png\n",
      "4\n",
      "https://imgs.xkcd.com/s/temperature.png\n",
      "4\n",
      "[<img alt=\"xkcd.com logo\" height=\"83\" src=\"/s/0b7742.png\" width=\"185\"/>, <img border=\"0\" src=\"//imgs.xkcd.com/static/how_to_news.png\"/>, <img alt=\"Landscape (sketch)\" src=\"//imgs.xkcd.com/comics/landscape_cropped_(1).jpg\" title=\"There's a river flowing through the ocean\"/>, <img alt=\"Selected Comics\" height=\"100\" src=\"//imgs.xkcd.com/s/a899e84.jpg\" usemap=\"#comicmap\" width=\"520\"/>, <img alt=\"Earth temperature timeline\" border=\"0\" height=\"100\" src=\"//imgs.xkcd.com/s/temperature.png\" width=\"520\"/>]\n",
      "/s/0b7742.png\n",
      "0\n",
      "https:/s/0b7742.png\n",
      "<urlopen error no host given>\n",
      "//imgs.xkcd.com/static/how_to_news.png\n",
      "1\n",
      "https://imgs.xkcd.com/static/how_to_news.png\n",
      "//imgs.xkcd.com/comics/landscape_cropped_(1).jpg\n",
      "2\n",
      "https://imgs.xkcd.com/comics/landscape_cropped_(1).jpg\n",
      "//imgs.xkcd.com/s/a899e84.jpg\n",
      "3\n",
      "https://imgs.xkcd.com/s/a899e84.jpg\n",
      "//imgs.xkcd.com/s/temperature.png\n",
      "4\n",
      "https://imgs.xkcd.com/s/temperature.png\n"
     ]
    }
   ],
   "source": [
    "i=0\n",
    "for n in range(1,5):\n",
    "    print(n)\n",
    "    a=requests.get('https://xkcd.com/'+str(n))\n",
    "    soup=bs(a.content)\n",
    "    images=soup.find_all('img')\n",
    "    print(images)\n",
    "    for i,img in enumerate(images):\n",
    "        print(img['src'])\n",
    "        url='https:'+img['src']\n",
    "        print(i)\n",
    "        print(url)\n",
    "        try:\n",
    "            urllib.request.urlretrieve(url,'Desktop/Sabudh/images/'+str(i)+str(n)+'.png')\n",
    "        except Exception as err:\n",
    "            print(err)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Desktop/Sabudh/images/test1.png', <http.client.HTTPMessage at 0x16eca15a3c8>)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import urllib.request\n",
    "urllib.request.urlretrieve('https://imgs.xkcd.com/comics/inbox.png', 'Desktop/Sabudh/images/test1.png')    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}