<jupyter_start><jupyter_text>Prepare Tourist Site data<jupyter_code># import needed libraries
import requests
from bs4 import BeautifulSoup
import time
import pandas as pd<jupyter_output><empty_output><jupyter_text>Web scraping www.tripadvisor.com.ph to extract tourist sites data.Disclaimers: - All of the data extracted are not my own and are properties of www.tripadvisor.com.ph- Scraped data are not used for commercial purposes and purely for personal education purposes- HTML, format, tags, parameters, and other website script used as reference are working as of this writing and may change anytime by the website owner/administrators which may impact this code- The code blocks may take 0-2mins depending on hardware/software/network capabilities. If you want to rerun, patience is appreciated- Imported time module to avoid overloading the site and get blocked  Use Search Url and create BeautifulSoup object<jupyter_code># set headers for scraping
headers = {
        'Access-Control-Allow-Origin': '*',
        'Access-Control-Allow-Methods': 'GET',
        'Access-Control-Allow-Headers': 'Content-Type',
        'Access-Control-Max-Age': '3600',
        "User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36"
    }

# get html text and create BeautifulSoup object 
search_url = 'https://www.tripadvisor.com.ph/Attractions-g298449-Activities-a_allAttractions.true-zft11309-Metro_Manila_Luzon.html'
search_html = requests.get(search_url, headers = headers)    
search_soup = BeautifulSoup(search_html.text, 'html.parser')
# extract tourist site links based from the search result page

# Use 3 lines below this to find the link_class 
#a_tags = search_soup.find_all('a') 
#for tag in a_tags :
    #print(tag)

links_class = {'class' : 'FmrIP _R w _Z P0 M0 Gm ddFHE'}
a_tags = search_soup.find_all('a', links_class) # find a tags
main_url = 'https://www.tripadvisor.com.ph'

# iterate and append all links in a list
tour_links_lst = []
for tag in a_tags :
    tour_links_lst.append(main_url + tag.get('href') ) 
     
print('Scraping Complete!')
print('Count of Tourist Site Links : ', len(tour_links_lst) )
print('Sample Links:')
for link in tour_links_lst[0:3] : print(link)
# iterate over each tourist site link to scrape the data of each 
tour_main_dict = {}
for tour_link in tour_links_lst :
    tour_details_dict = {}
    tour_html = requests.get(tour_link, headers = headers)    
    tour_soup = BeautifulSoup(tour_html.text, 'lxml')
    
    # Extract attraction name and add to details dictionary
    name = tour_soup.find('h1', {'class' : 'WlYyy cPsXC GeSzT'} ).get_text()
    tour_details_dict['tourist_site_name'] = name
    
    # Extract tourist site classification and add to details dictionary
    type_tag = tour_soup.find_all('div', {'class' : 'WlYyy diXIH dDKKM'})
    for tag in type_tag[4:5] :
        tour_classification = tag.get_text()
    tour_details_dict['tourist_site_classification'] = tour_classification
    
    # Extract tourist site location and add to details dictionary
    loc_tag = tour_soup.find('div', {'class' : 'dIDBU MJ'} )
    if loc_tag != None :
        tour_loc = loc_tag.get_text().split('Address')[-1]
    else : 
        tour_loc = tour_link.split('-')[-1].replace('_', ' ').split('.html')[0] 
    
    tour_details_dict['tourist_site_location'] = tour_loc

    # Extract tourist site other info (overall rating) and add to details dictionary
    other_tag = tour_soup.find_all('div', {'class' : 'WlYyy cPsXC fksET cMKSg'} )
    for tag in other_tag :
        other_info = tag.get_text() + '/5.0'
    tour_details_dict['tourist_site_other_info_overall_rating'] = other_info
    
    # Extract reviews and add to details dictionary
    reviews_tag = tour_soup.find_all('div', {'class' : 'pIRBV _T KRIav'})
    rev_count = 0
    tour_review = ''
    for tag in reviews_tag[1:4] :
        if tag.get_text() == '' : continue
        rev_count += 1
        tour_review = tour_review = tour_review + '\n' + 'Review #' + str(rev_count) + '\n' + tag.get_text()
    tour_details_dict['tourist_site_reviews'] = tour_review.lstrip()
    tour_main_dict[name] = tour_details_dict # add all scraped details of each tourist site
    time.sleep(1)
    
print('Scraping Complete!')
print('Count of Hotels:', len(tour_main_dict))
# Create Dataframe from tour_main_dict
df = pd.DataFrame.from_dict(tour_main_dict, orient = 'index')
df.info()
df.head(1)
# lean location data
def clean_loc(s) :
    if ',' not in s :
        r = s.split(' ')[0]
    else : 
        r = s.split(',')[-2]
    return r.strip() + ' City'
     
df['tourist_site_location'] = df['tourist_site_location'].apply(clean_loc)
df.tourist_site_location.loc[df['tourist_site_location'] == 'Quezon City City'] = 'Quezon City'
df.tourist_site_location.loc[df['tourist_site_location'] == 'San City'] = 'San Juan City'
df.tourist_site_location.loc[df['tourist_site_location'] == 'Taguig City City'] = 'Taguig City'
#for loc in df.tourist_site_location.values :
    #print(loc)

df.tourist_site_location.value_counts()
# Number of Visitors per year and entrance fee are based on research that are available
# PSA, or other review site doesn't usually provide these data
# See tourist_site_visitors_and_price.csv for references

import_df = pd.read_csv('tourist_site_visitors_and_price.csv', index_col = 'tourist_site_name')
import_df.head(3)
import_df = import_df.drop(['references'], axis = 1)
import_df.head(3)          
# join 2 df, save to csv
df.set_index('tourist_site_name', inplace = True)
final_df = df.merge(import_df, how = 'left', on = 'tourist_site_name' )
final_df.info()
final_df
final_df.to_csv('Tourist_site.csv', index = True)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup