# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.common.action_chains import ActionChains
import urllib
from bs4 import BeautifulSoup
import time
import csv
import re
import sys
import os
import numpy as np
import pandas as pd
from IPython.core.debugger import set_trace

path = 'datasets/alltrails/'

chrome_options = webdriver.chrome.options.Options()
chrome_options.add_argument("--start-maximized")
#chrome_options.add_argument('--headless')
#chrome_options.add_argument('--no-sandbox')
#chrome_options.add_argument('--disable-dev-shm-usage')
chrome_path = r'C:\Users\Sumit\Anaconda3\pkgs\python-chromedriver-binary-77.0.3865.40.0-py36_0\Lib\site-packages\chromedriver_binary\chromedriver.exe'

import pickle
infile=open('AllTrailsScraper_notebook_env.db','rb')
df_trails = pickle.load(infile)
infile.close()


def login():
    driver.get("https://www.alltrails.com/login?ref=header#")
    email = driver.find_element_by_id("user_email").send_keys("<EMAIL>")
    password = driver.find_element_by_id("user_password").send_keys("<PASSWORD>")
    login_button = driver.find_element_by_class_name("login")
    login_button.click()


def scrape_trail(browser, hike_url):
    browser.get(hike_url)
    soup = BeautifulSoup(browser.page_source, "lxml")
    return soup


# +
# Below processing should already have been done:
# df_trails['trail'] = df_trails['trail'].str.split('?').str[0]
# df_trails['trail_url'] = df_trails['trail_url'].str.split('?').str[0]
# df_trails['trail_url'] = df_trails['trail_url'].str.replace('/explore/','/')

# +
driver = webdriver.Chrome(chrome_path, options=chrome_options)
login()
time.sleep(1)

df_trail_latlon = pd.DataFrame()

for index, row in df_trails.iterrows():
    park = row['park']
    trail = row['trail']
    trail_url = row['trail_url']
    print(trail_url)

    soup = scrape_trail(driver, trail_url)
    #print(soup.getText())
    
    e_trail_lat = soup.find('meta', {'property': 'place:location:latitude'})
    e_trail_lon = soup.find('meta', {'property': 'place:location:longitude'})
    trail_lat = float(e_trail_lat['content'])
    trail_lon = float(e_trail_lon['content'])
    print(trail_lat)
    print(trail_lon)
    
    df_trail_latlon = df_trail_latlon.append({'park':park,'trail':trail,'trail_url':trail_url,'trail_lat':trail_lat,'trail_lon':trail_lon}, ignore_index=True)

    # e_rev = soup.findAll('div', {'itemprop': 'review'})
    # print('Reviews: '+str(len(e_rev)))
    # for review in e_rev:
    #     e_rev_date = review.find('meta', {'itemprop': 'datePublished'})
    #     e_rev_body = review.find('p', {'itemprop': 'reviewBody'})
    #     rev_date = e_rev_date['content']
    #     rev_body = e_rev_body.getText()
    #     # print(rev_date)
    #     # print(rev_body)
    #     df_trail_reviews = df_trail_reviews.append({'park':park,'trail':trail,'date':rev_date,'review':rev_body}, ignore_index=True)
    # df_trail_reviews.to_csv(path+'reviews_'+park+'_'+trail+'.csv')
driver.quit()
# -

df_trail_latlon.to_csv(path+'traillatlon.csv')

df_trail_latlon



StarcoderdataJupyterScriptsDedupFiltered