<jupyter_start><jupyter_text>Selenium Web Scraping and Browsing Automation<jupyter_code>import IPython
from IPython.display import IFrame, Javascript, display

# selenium-python docs
url_selenium_python_docs = IFrame(
    src='https://selenium-python.readthedocs.io/',
    width='100%',
    height='600'
)

display(url_selenium_python_docs)
import selenium.webdriver

dir(selenium.webdriver)
# !which chromedriver geckodriver safaridriver operadriver msedgedriver

#### Note: In your machine, locate where your drivers were installed
from selenium.webdriver import Chrome, Firefox, Safari, Opera, Edge

import time
from selenium import webdriver

def get_chrome():
    return Chrome(executable_path='/usr/local/bin/chromedriver')

def get_firefox(headless=False):
    if headless:
        options = webdriver.FirefoxOptions()
        options.add_argument('--headless')
        return Firefox(executable_path='/usr/local/opt/ruby/bin/geckodriver', options=options)
    return Firefox(executable_path='/usr/local/opt/ruby/bin/geckodriver')

def get_safari():
    return Safari(executable_path='/usr/bin/safaridriver')

def get_opera():
    return Opera(executable_path='/usr/local/bin/operadriver')

def get_msedge():
    return Edge(executable_path='/Users/rodelarenas/Desktop/DEV/edgedriver_mac64/msedgedriver')
# Basic Navigation
BASE_URL = 'https://google.com'

# open a webdriver
firefox = get_firefox()

# maximize, minimize
firefox.maximize_window()
time.sleep(1)

# browsing
firefox.get(BASE_URL)
time.sleep(2)

firefox.get('https://www.gmanmi.com')
time.sleep(2)

# F / B
firefox.back()
time.sleep(2)

firefox.forward()
time.sleep(5)

# firefox.close()
firefox.quit()
# Basis Interaction
from selenium.webdriver.common.keys import Keys

dir(Keys)
BASE_URL = 'https://trends.google.com/trends/trendingsearches/daily?geo=PH'

firefox = get_firefox()
firefox.get(BASE_URL)
time.sleep(.2)

firefox.maximize_window()

dataset = []
for _ in range(5):
    try:
        # pause
        time.sleep(2)

        # element should be visible from the screen before you apply your action/s
        firefox.execute_script('window.scrollBy(0, document.body.scrollHeight)', '') # need to scroll the page to find the element

        # locate the element
        load_more_class = 'feed-load-more-button'
        load_more_button = firefox.find_element_by_class_name(load_more_class)

        # do the interaction
        load_more_button.click()

        # get the feed wrapper
        feed_wrapper_class = 'feed-list-wrapper'
        feeds = firefox.find_elements_by_class_name(feed_wrapper_class)

        for fw in feeds:
            # dates
            dates_class = 'content-header-title'
            dates = fw.find_element_by_class_name(dates_class)
            feed_date = dates.text

            # titles
            titles_class = 'title'
            titles = fw.find_elements_by_class_name(titles_class)
            txt_titles = [t.text for t in titles]

            # summary texts
            summary_text_class = 'summary-text'
            summary_text =  fw.find_elements_by_class_name(summary_text_class)
            txt_summaries = [s.text for s in summary_text]

            # sources
            source_time_class = 'source-and-time'
            sources =  fw.find_elements_by_class_name(source_time_class)
            txt_sources = [sp.text for sp in sources]

            for idx, _ in enumerate(txt_titles):
                title = txt_titles[idx]
                summary_text = txt_summaries[idx]
                sources_text = txt_sources[idx]
                sources_text = sources_text.split()
                source_page = sources_text[0]
                posted_date_time = sources_text[2] + ' ' + sources_text[3]

                data = {
                    'date': feed_date,
                    'title': title,
                    'summary_text': summary_text,
                    'source_page': source_page,
                    'posted_date_time': posted_date_time
                }

                dataset.append(data)

    except:
        pass



import pandas as pd
df = pd.DataFrame(dataset)
df
df.to_csv('scraped_data.csv', index=False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup