Web Scraping with Selenium.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selenium web automation\n",
    "\n",
    "### Selenium is a suite of tools to automate web browsers across many platforms.\n",
    "#### https://www.seleniumhq.org/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Docs\n",
    "http://selenium-python.readthedocs.io/installation.html#downloading-python-bindings-for-selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip list\n",
    "# lists all packages installed via pip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "# Keys are shortcuts to common user navigation  keys UP, DOWN etc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We need to get an appropriate driver for browser and OS\n",
    "\n",
    "https://pypi.org/project/selenium/#description\n",
    "\n",
    "geckodriver.exe for Firefox on Windows https://github.com/mozilla/geckodriver/releases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Firefox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "drvChrome = webdriver.Chrome()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "drvChrome.get(\"https://www.ss.com/lv/real-estate/flats/riga/teika/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "mytable = drvChrome.find_element_by_id('page_main')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myrows = mytable.find_elements_by_tag_name('tr')\n",
    "len(myrows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "myrow = myrows[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mycells = myrow.find_elements_by_tag_name('td')\n",
    "len(mycells)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Mājīgs, mēbelēts dzīvoklis ar zemiem komunālajiem maksājumiem.\n",
      "Bērzpils 7\n",
      "1\n",
      "30\n",
      "4/5\n",
      "Hrušč.\n",
      "967 €\n",
      "29,000 €\n"
     ]
    }
   ],
   "source": [
    "for el in mycells:\n",
    "    print(el.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "mylst=[]\n",
    "for el in myrows[4:]:\n",
    "    cells = el.find_elements_by_tag_name('td')\n",
    "    if len(cells) > 3:\n",
    "        mylst.append({'Desc':cells[2].text,'Street':cells[3].text})\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mylst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "ename": "StaleElementReferenceException",
     "evalue": "Message: stale element reference: element is not attached to the page document\n  (Session info: chrome=67.0.3396.99)\n  (Driver info: chromedriver=2.38.552522 (437e6fbedfa8762dec75e2c5b3ddb86763dc9dcb),platform=Windows NT 6.1.7600 x86_64)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mStaleElementReferenceException\u001b[0m            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-85-64f13c5c0391>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmyrows\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36mclick\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     78\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     79\u001b[0m         \u001b[1;34m\"\"\"Clicks the element.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 80\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_execute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCLICK_ELEMENT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0msubmit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36m_execute\u001b[1;34m(self, command, params)\u001b[0m\n\u001b[0;32m    626\u001b[0m             \u001b[0mparams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    627\u001b[0m         \u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 628\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    629\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    630\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    312\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    313\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 314\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    315\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    316\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mStaleElementReferenceException\u001b[0m: Message: stale element reference: element is not attached to the page document\n  (Session info: chrome=67.0.3396.99)\n  (Driver info: chromedriver=2.38.552522 (437e6fbedfa8762dec75e2c5b3ddb86763dc9dcb),platform=Windows NT 6.1.7600 x86_64)\n"
     ]
    }
   ],
   "source": [
    "myrows[4].click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "myrows[4].find_elements_by_tag_name('td')[2].click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mājīgs, mēbelēts dzīvoklis ar zemiem komunālajiem maksājumiem. Bērzpils 7\n",
      "Tikko pēc kapitāla eiroremonta, un ar visu nepieciešamu premium Zālīša 1\n",
      "Место - вне конкуренции. центр \"старой\" Тейки. Напротив Teikas V Āraišu 2\n",
      "Tiek izīrēts 2-istabu dzīvoklis, ar izolētām istabām ķieģeļu, sp Ūnijas 28\n",
      "Īpašnieks izīrē. saulains, gaišs, daļēji Mēbelēts(Ikea), Necaurs Piebalgas 4\n",
      "Unikāls dzīvokļu projekts izvietots slēgtā, zaļā teritorijā ar d Vairoga 32\n",
      "Unikāls dzīvokļu projekts izvietots slēgtā, zaļā teritorijā ar d Vairoga 32\n",
      "Unikāls dzīvokļu projekts izvietots slēgtā, zaļā teritorijā ar d Vairoga 32\n",
      "Уникальный проект с детской площадкой и парковкой на закрытой зе Vairoga 32\n",
      "Īpašniece izīrē mājīgu, vienas istabas studio tipa dzīvokli (15 Struktoru 13\n",
      "Pārdod pilnībā labiekārtotu dzīvokli projektā ''Jaunā Teika''. Ropažu 10\n",
      "Piedāvājam iegādāties jauku, saulainu vienistabas dzīvokli privā Zālīša 3\n",
      "Šobrīd aktīvi meklējam savu klientu interesēs divu līdz četru (2 Ieriķu 29\n",
      "Piedāvājam iegādāties jauku, kvalitatīvi remontētu divistabu dz Laimdotas 57\n",
      "Pārdodu dzīvokli divstāvu mājā, ar izolētām istabām. Junkers gāz Ropažu 2\n",
      "Saimnieks izīrē mēbelētu divistabu dzīvokli ilgtermiņā. Ir balko Biķernieku 79\n",
      "Редкое предложение. Просторная и светлая 3-х комнатная квартир Brīvības 241\n",
      "Firma steidzami vēlas īrēt divistabu dzīvokli Teikā vai Čiekurka -\n",
      "Pārdošanā lielisks 2-istabu dzīvoklis paša Teika sirdī. Slēgta k Brīvības 326\n",
      "За наличные деньги - покупаем - 1 - 1.5 - 2 - 3 - 4 - ком. к - 777\n",
      "Piedāvājam mājīgu divistabu mansarda dzīvoklīti. Īpašums atrod Ropažu 91\n",
      "Namīpašnieks izīrē dzīvokli pēc remonta. Vietējā gāzes apkure. Pudiķa 13\n",
      "Piedāvājam iegādāties gaišu, plašu dzīvokli Teikā, labā vietā Br Brīvības 358\n",
      "Dzīvoklis labā tehniskā un vizuālā stāvoklī, neprasa steidzamus Ropažu 122 k-5\n",
      "Netālu no Alfas, izīrē īpašnieks Brīvības 334\n",
      "Продаётся квартира. Квартира в середине дома. Очень тёплая, солн Burtnieku 33\n",
      "Izīrē divistabas dzīvokli- izolētas istabas ar balkonu. Biķernieku 35A\n",
      "Tikko pēc remonta. Dzīvoklis absolūti perfektā stāvoklī, jo nevi Brīvības 262\n",
      "Совершенно эксклюзивный вариант. Трехэтажный довоенный особняк н Bērzaunes 3\n",
      "Īpašnieks pārdod remontējamu 1 istabas dzīvokli ļoti labā vietā, Biķernieku 47\n"
     ]
    }
   ],
   "source": [
    "for el in mylst:\n",
    "    print(el['Desc'],el['Street'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "StaleElementReferenceException",
     "evalue": "Message: stale element reference: element is not attached to the page document\n  (Session info: chrome=67.0.3396.99)\n  (Driver info: chromedriver=2.38.552522 (437e6fbedfa8762dec75e2c5b3ddb86763dc9dcb),platform=Windows NT 6.1.7600 x86_64)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mStaleElementReferenceException\u001b[0m            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-43-821d53f477b5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmyprice\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36mtext\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     74\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     75\u001b[0m         \u001b[1;34m\"\"\"The text of the element.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 76\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_execute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mGET_ELEMENT_TEXT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     77\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     78\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36m_execute\u001b[1;34m(self, command, params)\u001b[0m\n\u001b[0;32m    626\u001b[0m             \u001b[0mparams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    627\u001b[0m         \u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 628\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    629\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    630\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    312\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    313\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 314\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    315\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    316\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mStaleElementReferenceException\u001b[0m: Message: stale element reference: element is not attached to the page document\n  (Session info: chrome=67.0.3396.99)\n  (Driver info: chromedriver=2.38.552522 (437e6fbedfa8762dec75e2c5b3ddb86763dc9dcb),platform=Windows NT 6.1.7600 x86_64)\n"
     ]
    }
   ],
   "source": [
    "myprice.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "ename": "WebDriverException",
     "evalue": "Message: unknown command: session/e67abc7ea000cbe737b4a283de1bedc0/element/0.6160322096501127-3/screenshot\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mWebDriverException\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-38-0d651e93a7cf>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmyrow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscreenshot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'screenshot.png'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36mscreenshot\u001b[1;34m(self, filename)\u001b[0m\n\u001b[0;32m    578\u001b[0m             warnings.warn(\"name used for saved screenshot does not match file \"\n\u001b[0;32m    579\u001b[0m                           \"type. It should end with a `.png` extension\", UserWarning)\n\u001b[1;32m--> 580\u001b[1;33m         \u001b[0mpng\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscreenshot_as_png\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    581\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    582\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36mscreenshot_as_png\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    560\u001b[0m             \u001b[0melement_png\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0melement\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscreenshot_as_png\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    561\u001b[0m         \"\"\"\n\u001b[1;32m--> 562\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mbase64\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mb64decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscreenshot_as_base64\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'ascii'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    563\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    564\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mscreenshot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36mscreenshot_as_base64\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    550\u001b[0m             \u001b[0mimg_b64\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0melement\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscreenshot_as_base64\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    551\u001b[0m         \"\"\"\n\u001b[1;32m--> 552\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_execute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mELEMENT_SCREENSHOT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'value'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    553\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    554\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36m_execute\u001b[1;34m(self, command, params)\u001b[0m\n\u001b[0;32m    626\u001b[0m             \u001b[0mparams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    627\u001b[0m         \u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 628\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    629\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    630\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    312\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    313\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 314\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    315\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    316\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    206\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mexception_class\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mErrorInResponseException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    207\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 208\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    209\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmessage\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"\"\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;34m'message'\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    210\u001b[0m             \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'message'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mWebDriverException\u001b[0m: Message: unknown command: session/e67abc7ea000cbe737b4a283de1bedc0/element/0.6160322096501127-3/screenshot\n"
     ]
    }
   ],
   "source": [
    "myrow.screenshot('screenshot.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-43\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-44\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-45\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-46\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-47\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-48\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-49\")>]\n"
     ]
    }
   ],
   "source": [
    "print(mycells)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-1\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-2\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-3\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-4\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-5\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-6\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-7\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-8\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-9\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-10\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-11\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-12\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-13\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-14\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-15\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-16\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-17\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-18\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-19\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-20\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-21\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-22\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-23\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-24\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-25\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-26\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-27\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-28\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-29\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-30\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-31\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-32\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-33\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-34\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-35\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-36\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-37\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-38\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-39\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-40\")>, <selenium.webdriver.remote.webelement.WebElement (session=\"e67abc7ea000cbe737b4a283de1bedc0\", element=\"0.6160322096501127-41\")>]\n"
     ]
    }
   ],
   "source": [
    "print(myrows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "myinp = driver.find_element_by_id('input')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "myinp.send_keys('really')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mysub = driver.find_element_by_id('submit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "mysub.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(\"https://www.ss.com/lv/real-estate/flats/riga/kengarags/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "myinp = driver.find_element_by_css_selector('input.s12')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "selenium.webdriver.firefox.webelement.FirefoxWebElement"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(myinp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "myinp.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "myback = driver.find_element_by"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "binary=\"C:\\\\Program Files (x86)\\\\Mozilla Firefox\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.firefox.firefox_binary import FirefoxBinary\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "binary = FirefoxBinary(\"C:\\\\Program Files (x86)\\\\Mozilla Firefox\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "binfox=FirefoxBinary( \"C:\\\\bin\\\\Gecko\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "browser = webdriver.Firefox(firefox_binary=binfox)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3;C:\\ProgramData\\Anaconda3\\Library\\mingw-w64\\bin;C:\\ProgramData\\Anaconda3\\Library\\usr\\bin;C:\\ProgramData\\Anaconda3\\Library\\bin;C:\\ProgramData\\Anaconda3\\Scripts;C:\\Program Files\\ImageMagick-7.0.7-Q16;C:\\ProgramData\\Anaconda3;C:\\ProgramData\\Anaconda3\\Library\\mingw-w64\\bin;C:\\ProgramData\\Anaconda3\\Library\\usr\\bin;C:\\ProgramData\\Anaconda3\\Library\\bin;C:\\ProgramData\\Anaconda3\\Scripts;C:\\ProgramData\\Oracle\\Java\\javapath;C:\\Windows\\system32;C:\\Windows;C:\\Windows\\System32\\Wbem;C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\;C:\\Program Files\\Intel\\WiFi\\bin\\;C:\\Program Files\\Common Files\\Intel\\WirelessCommon\\;C:\\Program Files\\WIDCOMM\\Bluetooth Software\\;C:\\Program Files\\WIDCOMM\\Bluetooth Software\\syswow64;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Intel\\WiFi\\bin\\;C:\\Program Files\\Common Files\\Intel\\WirelessCommon\\;C:\\Program Files\\Microsoft VS Code\\bin\n"
     ]
    }
   ],
   "source": [
    "! echo %PATH%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we copied geckodriver.exe to C:\\ProgramData\\Anaconda not pretty but it works!\n",
    "driver = webdriver.Firefox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MACOS driver = webdriver.Safari()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(\"https://fortnitetracker.com/leaderboards\")\n",
    "#http://www.dvi.gov.lv/pub/pub.php?f=spd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir(driver)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tables=driver.find_elements_by_tag_name(\"table\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir(tables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tables[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir(tables[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows=tables[0].find_elements_by_tag_name(\"tr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(rows[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cells=rows[1].find_elements_by_tag_name(\"td\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cells)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "first = [el.get_attribute('innerHTML') for el in cells]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name=cells[1].find_element_by_class_name(\"trn-lb-entry__name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name.get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cells = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows[4].find_elements_by_tag_name(\"td\")[1].find_element_by_class_name(\"trn-lb-entry__name\").get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows[5].find_elements_by_tag_name(\"td\")[1].find_element_by_class_name(\"trn-lb-entry__name\").text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Now we get all innerHTML found containing Names of top players\n",
    "cells = [el.find_elements_by_tag_name(\"td\")[1].find_element_by_class_name(\"trn-lb-entry__name\").text for el in rows[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cells"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#from bs4 import BeautifulSoup\n",
    "from selenium import webdriver\n",
    "browser=webdriver.Firefox() #vajag instalet ChromeDriver\n",
    "browser.get('http://www.dvi.gov.lv/pub/pub.php?f=spd#')#Loads a web page in the current browser session.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table=browser.find_element_by_id('rectable')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows=table.find_elements_by_tag_name('tr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c1=rows[0].find_elements_by_tag_name('td')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(c1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[el.text for el in c1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser.find_element_by_link_text(str(3)).click() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(2,4):#manuali ievadi lapu skaitu\n",
    "    browser.get('http://www.dvi.gov.lv/pub/pub.php?f=spd#')\n",
    "#     browser.find_element_by_link_text(str(i)).click() \n",
    "#     time.sleep(1)\n",
    "    table=browser.find_element_by_id('rectable')\n",
    "    rows=table.find_elements_by_tag_name('tr')\n",
    "    for row in rows:\n",
    "        cells=row.find_elements_by_tag_name('td')\n",
    "        spec = [el.text for el in cells]\n",
    "        print(spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import csv\n",
    "import time\n",
    "try:\n",
    "    for i in range(2,4):#manuali ievadi lapu skaitu\n",
    "        browser.find_element_by_link_text(str(3)).click() \n",
    "#         browser.find_element_by_link_text(str(3)).click()\n",
    "        time.sleep(1)\n",
    "        pageSource=browser.page_source\n",
    "        table=pageSource.find_element_by_id('rectable')\n",
    "        rows=table.find_elements_by_tag_name('tr')\n",
    "        for row in rows:\n",
    "            cells=row.find_elements_by_tag_name('td')\n",
    "            spec = [el.text for el in cells]\n",
    "            print(spec)\n",
    "#         for spec in bsObj2.find_all(name='td'):#Extracts a list of Tag objects that match the given\n",
    "#             zx=spec.get_text(separator='|', strip=('title'))\n",
    "#             with open('specialisti_test.txt', mode='a') as teksts:\n",
    "#                 teksts.write(f'{zx}\\n')\n",
    "except NoSuchElementException as Err:\n",
    "    print(f'Kluda: ({Err})')\n",
    "    #pass\n",
    "except Exception as err:\n",
    "    print(f'Unknown exception ({err})')\n",
    "# browser.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative SCRAPY\n",
    "### https://scrapy.org/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}