discoverorg.ipynb

#Packages used in this program
#Cell No.1
import SeleniumLibrary
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from robot.libraries.BuiltIn import BuiltIn
from selenium.webdriver.common.keys import Keys
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.common.exceptions import ElementNotVisibleException
from selenium.common.exceptions import ElementClickInterceptedException
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import StaleElementReferenceException
import pandas as pd

#All the xpath used in the file.
#Cell No.2
loginbutton = "//button[@id='SignIn']"
username = "//input[@id='username']"
password = "//input[@id='loginPassword']"
UN = "xxxxxxx"
PW = "xxxxxxx"
rstab = "//a[@id='Search-RecentSearches']"
listpaper = "//div[@class='modal-header ng-scope']"
listselected = "/html[1]/body[1]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/select[1]/option[31]" #Change the number in option[] to select the list. 
applysearch = "//button[@id='Search-RecentSearches-ApplyButton']"
mapbutton = "//button[@class='btn btn-primary view-map-btn ng-binding']"
companybutton = "//button[@id='Search-Results-Companies']"
updatecom = "/html[1]/body[1]/div[1]/div[1]/div[1]/span[1]/ng-include[1]/div[1]/div[1]/form[1]/div[2]/label[1]/input[1]"
uploadnewcom = "/html[1]/body[1]/div[1]/div[1]/div[1]/span[1]/ng-include[1]/div[1]/div[1]/form[1]/div[1]/label[1]/input[1]"
resultbody = "/html[1]/body[1]/section[1]/section[1]/article[1]/div[2]/div[2]/div[1]/div[1]/div[1]/div[3]"
uploadbutton = "//a[@class='btn btn-primary upload-upload-btn ng-binding ng-scope']"
cancelbutton = "//a[@class='btn btn-default upload-cancel-btn']"
uppapertext = "//th[contains(text(),'DiscoverOrg')]"
nextpage = "//a[contains(text(),'Next')]"

#Create a function for the login process and redirect to the result page
#Cell No.3
def loginpage(username,password,UN,PW):
    driver = webdriver.Chrome()
    driver.maximize_window()
    driver.get("https://go.discoverydb.com/eui/#/login")
    loginpage = WebDriverWait(driver, 10).until(ec.visibility_of_element_located((By.XPATH, loginbutton)))
    uninput = driver.find_element_by_xpath(username)
    uninput.send_keys(UN)
    pwinput = driver.find_element_by_xpath(password)
    pwinput.send_keys(PW)
    loginpage.click()
    time.sleep(5)
    changetab = WebDriverWait(driver, 20).until(ec.visibility_of_element_located((By.XPATH, rstab)))
    time.sleep(5)
    changetab.click()
    selectsr = WebDriverWait(driver,20).until(ec.visibility_of_element_located((By.XPATH,listpaper)))
    time.sleep(2)
    targetsr = driver.find_element_by_xpath(listselected)
    targetsr.location_once_scrolled_into_view
    targetsr.click()
    asbutton = driver.find_element_by_xpath(applysearch)
    asbutton.click()
    time.sleep(3)
    companys = WebDriverWait(driver,10).until(ec.visibility_of_element_located((By.XPATH,companybutton)))
    companys = WebDriverWait(driver,10).until(ec.element_to_be_clickable((By.XPATH,companybutton)))
    companys.click()
    tablebody = WebDriverWait(driver,30).until(ec.visibility_of_element_located((By.XPATH,mapbutton)))
    return driver

#Call loginpage function
#Cell No.4
driver = loginpage(username,password,UN,PW)

#Record all the companies need an upload action, as well as the companies updated
#Cell No.5
comlist = []
updatelist = []
#Find the total number of pages (100 per page)
comnum = driver.find_element_by_xpath("//a[@id='Search-ActiveCriteria']//span[@class='badge ng-binding']").text
comnum = int(comnum)
rnum = int(comnum//100)

#Create a function to locate the 100 companies in current page one by one.
#Cell No.6
def cloudshow(i):
    cloud = "/html[1]/body[1]/section[1]/section[1]/article[1]/div[2]/div[2]/div[1]/div[1]/div[1]/div[4]/div[1]/div[2]/div[1]/div[%i]/div[1]/div[1]/div[1]/div[1]/i[3]"%i
    cloudbutton = WebDriverWait(driver,60).until(ec.visibility_of_element_located((By.XPATH,cloud)))
    cloudbutton = driver.find_element_by_xpath(cloud)
    cloudbutton.location_once_scrolled_into_view
    time.sleep(1)
    cloudbt = WebDriverWait(driver,60).until(ec.element_to_be_clickable((By.XPATH, cloud)))
    return cloud,cloudbt

#Create a fundtion to update company records one by one in current page.
#Cell No.7
def updaterecordbypage(start):
    for i in range(int(start),101):
        rbody = WebDriverWait(driver,60).until(ec.visibility_of_element_located((By.XPATH,resultbody)))
        driver.execute_script("window.scrollBy(0,30)")
        cloud,cloudbt = cloudshow(i)

        try:
            cloudbuttonon = cloudbt
        except NoSuchElementException or StaleElementReferenceException:
            time.sleep(20)
            cloudbutton = WebDriverWait(driver,100).until(ec.element_to_be_clickable((By.XPATH, cloud)))
            cloudbutton = cloudbt
            
        try:
            cloudbt.click()
        except ElementClickInterceptedException:
            driver.execute_script("window.scrollBy(0,30)")
            time.sleep(10)
            cloudbt = WebDriverWait(driver,60).until(ec.element_to_be_clickable((By.XPATH, cloud)))
            cloudbt.click()
            
        uploadcom = WebDriverWait(driver,60).until(ec.visibility_of_element_located((By.XPATH,uploadnewcom)))
        updatepage = driver.find_element_by_xpath(updatecom)
        uploadcomname = driver.find_element_by_xpath("//label[contains(text(),'Upload New Company')]").text
        
        try:
            updatepage.click()
            updatelist.append(str(uploadcomname))
            updateshow = WebDriverWait(driver,60).until(ec.element_to_be_clickable((By.XPATH, cloud)))
            time.sleep(2)
        except ElementNotVisibleException:
            comlist.append(str(uploadcomname))
            time.sleep(2)
            
        targetbt = driver.find_element_by_xpath(uploadbutton)
        targetbt.location_once_scrolled_into_view  
        dis_data = targetbt.get_attribute('disabled')
        
        if dis_data is None:
            is_clickable = True
        else:
            is_clickable = False
            
        if is_clickable == False:
            tarcancel = driver.find_element_by_xpath(cancelbutton)
            tarcancel.click()
        else:
            targetbt.click()
            
        time.sleep(2)
        record = i
    pagenum = driver.find_element_by_xpath("//li[@class='pagination-page ng-scope active']").text
    return comlist,updatelist,record

#Create a function to the next page.
#Cell No.8
def tonextpage(current):
    c = current
    nextbutton = driver.find_element_by_xpath(nextpage)
    driver.execute_script("return arguments[0].scrollIntoView(true);", nextbutton)
    nextbutton = WebDriverWait(driver,30).until(ec.element_to_be_clickable((By.XPATH,nextpage)))
    nextbutton.click()
    pagenum = driver.find_element_by_xpath("//li[@class='pagination-page ng-scope active']").text
    p = int(pagenum)
    time.sleep(2)
    return p

#Create a function to the destinate page.
#Cell No.9
def updatepage(startpage):
    for i in range(int(startpage),rnum):
        pagenum = driver.find_element_by_xpath("//li[@class='pagination-page ng-scope active']").text
        if int(pagenum) != int(startpage):
            p = tonextpage(1)
            
#Create a funtion to gather all other functions to work.
#Cell No.10
def updatebypage(record):        
    try: 
        newpage = WebDriverWait(driver,30).until(ec.visibility_of_element_located((By.XPATH,companybutton)))
        uploadlist,updatelist,re = updaterecordbypage(int(record))
        time.sleep(5)
        tonextpage(1)
        pagenum = driver.find_element_by_xpath("//li[@class='pagination-page ng-scope active']").text
        p = int(pagenum)
        print("Checking %ith page"%p)
        nextbutton = WebDriverWait(driver,30).until(ec.element_to_be_clickable((By.XPATH,nextpage)))
        dis_next = nextbutton.get_attribute('disabled')
        
        if dis_next is None:
            is_clickable = True
        else:
            is_clickable = False
            
        if is_clickable == False:
            print("Last page") 
            time.sleep(2)
        else:
            tonextpage(1)

    except TimeoutException or ElementClickInterceptedException:
        print("Get stuck on page %i the %i record"%p%re)
        driver.delete_all_cookies()
        driver.close()
        pd.DataFrame(uploadlist).to_excel('uploadlist.xlsx', header=False, index=False)
        pd.DataFrame(updatelist).to_excel('updatelist.xlsx', header=False, index=False)
    return p,re,uploadlist,updatelist

#Run from the first page
#Cell No.11
updatepage(1)
pn,re,list1,list2 = updatebypage(1)

#Run from the 'get stuck page and record' from Cell No.11.
#Cell No.12
loginpage(username,password,UN,PW)
updatepage(pn)
pn2,re2,list3,list4 = updatebypage(re)