Selenium
Using Selenium to scrape inventory data#
I use this method if an API doesn't exist.
import time as tfrom selenium.webdriver.support.ui import Selectfrom datetime import timefrom selenium import webdriverfrom selenium.webdriver.common.keys import Keysfrom selenium.webdriver.support.ui import WebDriverWaitfrom webdriver_manager.chrome import ChromeDriverManagerimport csvfrom selenium.common.exceptions import NoSuchElementException, WebDriverException, InvalidSessionIdExceptionfrom selenium.common.exceptions import ElementNotInteractableException, ElementClickInterceptedExceptionfrom selenium.webdriver.chrome.options import Options
columns = ['style_num', 'size', 'nike_quantities']
def quantities(): chrome_options = Options() chrome_options.add_argument("--disable-extensions") chrome_options.add_argument("--disable-gpu") chrome_options.add_argument("--window-size=1920x1080") chrome_options.add_argument("--no-sandbox") # linux only chrome_options.add_argument("--headless") chrome_options.headless = True # also works # credentials username = "" password = "" # initialize the Chrome driver driver = webdriver.Chrome(ChromeDriverManager().install()) # driver = webdriver.Chrome()
# head to login page driver.get("") driver.implicitly_wait(10) # find username/email field and send the username itself to the input field driver.find_element_by_id("okta-signin-username").send_keys(username) # find password input field and insert password as well driver.find_element_by_id("okta-signin-password").send_keys(password) # click login button driver.find_element_by_id("okta-signin-submit").click() driver.find_element_by_class_name("css-1yfebrz.ec98wd50").click() t.sleep(4) driver.find_element_by_id("PRTL_LNK_CHOOSE").click() t.sleep(2) driver.find_element_by_id("PRTL_LNK_ATON").click() t.sleep(4) driver.find_element_by_id("shop-at-once").click() t.sleep(4)
error_message = "Incorrect username or password." # get the errors (if there are) errors = driver.find_elements_by_class_name("flash-error") # if we find that error message within errors, then login is failed if any(error_message in e.text for e in errors): print("[!] Login failed") else: print("[+] Login successful")
with open('./uploads/style.csv', 'r') as file: reader = csv.reader(file, delimiter=",") header = next(reader) with open('./results/results.csv', 'w') as f: csv_writer = csv.writer(f, lineterminator = '\n') csv_writer.writerow(columns)
# iterate through csv for row in reader: style_num = row[0] size = row[1]
try: print('--->', style_num, size) except: continue try:
searchterm = style_num # edit me size = size # iterate through size data
# if product doesn't exist send error and refresh browser page displayed_error = driver.find_element_by_class_name('search-error') error_msg = 'no items' if displayed_error: driver.refresh() print(error_msg) else: continue
# add style_num to search box driver.find_element_by_id("shop-at-once").click() t.sleep(4) driver.get('') # clear_box = driver.find_element_by_id("search").clear() sbox = driver.find_element_by_class_name("search-input") t.sleep(4) sbox.send_keys(searchterm) t.sleep(4)
# click to search driver.find_element_by_class_name("search-btn.ng-scope").click()
# click on item driver.find_element_by_class_name("can-grey").click() t.sleep(4)
# copy and paste quantity info nike_qty = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[7]/div[2]/div[3]/div[1]/div[2]/div[4]/div/div[2]') nike_size = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[2]/div[7]/div[2]/div[3]/div[1]/div[2]/div[4]/div/div[1]') qty_data = nike_qty.text size_data = nike_size.text # no_products = no_units.text t.sleep(4) print(size_data) print(qty_data)
# create a list for string then create a dict:chrtist def parse_qty(input_str): input_str = qty_data items = input_str.split(' ') items = [item.replace('+', '') for item in items] return items
def parse_size(input_str): input_str = size_data items = input_str.split(' ') items = [f"'{item}'" for item in items] return items
qty_items = parse_qty(qty_data) size_items = parse_size(size_data) # print(size_items) # print(qty_items)
# create {key:value} for size and qty. a_dict = dict(zip(size_items, qty_items)) # print(a_dict) # add quotes to sting from csv file size_to_lookup = f"'{size}'" print(size_to_lookup)
if size_to_lookup in a_dict: print(a_dict[size_to_lookup]) else: print(size_to_lookup,'item is not found') csv_writer.writerow([searchterm, size, 'item is not found']) csv_writer.writerow([searchterm, size, a_dict[size_to_lookup]])
except NoSuchElementException: continue except ElementNotInteractableException: continue except InvalidSessionIdException: continue except ElementClickInterceptedException: continue except KeyError: continue finally: try: driver.find_element_by_class_name('back-button').click() except: pass
if __name__ == "__main__": quantities()