<jupyter_start><jupyter_text>Prepare Hotel Data<jupyter_code># import needed libraries
import requests
from bs4 import BeautifulSoup
import time
import pandas as pd<jupyter_output><empty_output><jupyter_text>Web scraping www.booking.com to extract hotels data. Disclaimers: - All of the data extracted are not my own and are properties of www.booking.com and data will not be used for commercial purposes- Scraped data are not used for commercial purposes and purely for personal education purposes- HTML, format, tags, parameters, and other website script used as reference are working as of this writing and may be changed anytime by the website owner/administrators which may impact this code- The code blocks may take 0-2mins depending on hardware/software/network capabilities. If you want to rerun, patience is appreciated- Imported time module to avoid overloading the site and get blocked<jupyter_code># Web Scraping: scrape details of each hotel from search results in www.booking.com 
headers = {
        'Access-Control-Allow-Origin': '*',
        'Access-Control-Allow-Methods': 'GET',
        'Access-Control-Allow-Headers': 'Content-Type',
        'Access-Control-Max-Age': '3600',
        "User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36"
    }
# get html text and create BeautifulSoup object 
url_5k_10k = 'https://www.booking.com/searchresults.html?label=gen173nr-1DCAEoggI46AdIM1gEaLQBiAEBmAExuAEXyAEM2AED6AEB-AECiAIBqAIDuAK-vIuQBsACAdICJDNiYTIzMGMwLTgyYjUtNGFiYS04MjlkLTk1ZjhkMDZjZGM4YdgCBOACAQ&sid=c181dd9af9df88eff960c65192ff13a9&aid=304142&src=searchresults&error_url=https%3A%2F%2Fwww.booking.com%2Fsearchresults.html%3Flabel%3Dgen173nr-1DCAEoggI46AdIM1gEaLQBiAEBmAExuAEXyAEM2AED6AEB-AECiAIBqAIDuAK-vIuQBsACAdICJDNiYTIzMGMwLTgyYjUtNGFiYS04MjlkLTk1ZjhkMDZjZGM4YdgCBOACAQ%3Bsid%3Dc181dd9af9df88eff960c65192ff13a9%3Btmpl%3Dsearchresults%3Bcheckin_monthday%3D01%3Bcheckin_year_month%3D2022-04%3Bcheckout_monthday%3D02%3Bcheckout_year_month%3D2022-04%3Bclass_interval%3D1%3Bdest_id%3D-2437894%3Bdest_type%3Dcity%3Bdtdisc%3D0%3Bfrom_history%3D1%3Bgroup_adults%3D2%3Bgroup_children%3D0%3Binac%3D0%3Bindex_postcard%3D0%3Blabel_click%3Dundef%3Bno_rooms%3D1%3Boffset%3D0%3Bpostcard%3D0%3Braw_dest_type%3Dcity%3Broom1%3DA%252CA%3Bsb_price_type%3Dtotal%3Bsb_travel_purpose%3Dleisure%3Bsh_position%3D1%3Bshw_aparth%3D1%3Bsi%3Dai%3Bsi%3Dci%3Bsi%3Dco%3Bsi%3Ddi%3Bsi%3Dla%3Bsi%3Dre%3Bslp_r_match%3D0%3Bsrpvid%3Db42595ebe16f00d7%3Bss_all%3D0%3Bssb%3Dempty%3Bsshis%3D0%26%3B&ss=Manila&is_ski_area=0&ssne=Manila&ssne_untouched=Manila&city=-2437894&checkin_year=2022&checkin_month=9&checkin_monthday=1&checkout_year=2022&checkout_month=9&checkout_monthday=2&group_adults=2&group_children=0&no_rooms=1&from_sf=1&nflt=price%3DPHP-5000-10000-1%3Bht_id%3D204&shw_aparth=0'
search_html_1 = requests.get(url_5k_10k, headers = headers)    
search_soup_1 = BeautifulSoup(search_html_1.text, 'html.parser')
# find all links in the soup object based on class attribute
hotel_link_class = {'class' :'_4310f7077 _45807dae0 _89c03ca2c _f7538b398'} 
hotels_1 = search_soup_1.find_all('a', hotel_link_class) 

count = 0 # for counting iterations
hotels_dict = {} # main dictionary of hotels data  
for hotel in hotels_1 : # iteration to put hotel details in a dict for each hotel
    hotel_details_dict = {} # dictionary of hotel data per hotel
    # extract hotel name
    hotel_name = hotel.get('href').strip()[:-6].split('/')[-1].split('.')[0]
    title_hotel_name = hotel_name.replace('-',' ').title()
    hotel_details_dict['hotel_name'] = title_hotel_name
    # price range category based on website search
    hotel_details_dict['hotel_min_price_range_php'] = '5k to 10k' 
    # extract reviews
    review_url = 'https://www.booking.com/reviews/ph/hotel/' + hotel_name + '.en-gb.html'
    review_html = requests.get(review_url, headers = headers)
    review_soup = BeautifulSoup(review_html.text, 'html.parser')
    reviewer_name = review_soup.find_all('p', {'class' : 'reviewer_name' } )
    review_content = review_soup.find_all('div', {'class' : 'review_item_review_content' } )
    zipped_reviews = dict(zip(reviewer_name, review_content) )
    review_count = 0
    for k, v in zipped_reviews.items() : # iterate to extract reviews with format
        review = 'Reviewer Name: {}\nReview: {}'.format(k.get_text(), v.get_text().strip().replace('\n', ' ') )
        review_count += 1
        if review_count == 3 : break
    hotel_details_dict['hotel_reviews'] = review 
    #extract hotel link
    hotel_details_dict['hotel_link'] = hotel.get('href').strip()[:-6]
    hotels_dict[hotel_name] = hotel_details_dict
    count += 1
    if count == 20 : break

# 2nd set of urls for different price range based on the website search
url_10k_up = 'https://www.booking.com/searchresults.html?label=gen173nr-1DCAEoggI46AdIM1gEaLQBiAEBmAExuAEXyAEM2AED6AEB-AECiAIBqAIDuAK-vIuQBsACAdICJDNiYTIzMGMwLTgyYjUtNGFiYS04MjlkLTk1ZjhkMDZjZGM4YdgCBOACAQ&sid=c181dd9af9df88eff960c65192ff13a9&aid=304142&src=searchresults&error_url=https%3A%2F%2Fwww.booking.com%2Fsearchresults.html%3Flabel%3Dgen173nr-1DCAEoggI46AdIM1gEaLQBiAEBmAExuAEXyAEM2AED6AEB-AECiAIBqAIDuAK-vIuQBsACAdICJDNiYTIzMGMwLTgyYjUtNGFiYS04MjlkLTk1ZjhkMDZjZGM4YdgCBOACAQ%3Bsid%3Dc181dd9af9df88eff960c65192ff13a9%3Btmpl%3Dsearchresults%3Bcheckin_monthday%3D01%3Bcheckin_year_month%3D2022-04%3Bcheckout_monthday%3D02%3Bcheckout_year_month%3D2022-04%3Bclass_interval%3D1%3Bdest_id%3D-2437894%3Bdest_type%3Dcity%3Bdtdisc%3D0%3Bfrom_history%3D1%3Bgroup_adults%3D2%3Bgroup_children%3D0%3Binac%3D0%3Bindex_postcard%3D0%3Blabel_click%3Dundef%3Bno_rooms%3D1%3Boffset%3D0%3Bpostcard%3D0%3Braw_dest_type%3Dcity%3Broom1%3DA%252CA%3Bsb_price_type%3Dtotal%3Bsb_travel_purpose%3Dleisure%3Bsh_position%3D1%3Bshw_aparth%3D1%3Bsi%3Dai%3Bsi%3Dci%3Bsi%3Dco%3Bsi%3Ddi%3Bsi%3Dla%3Bsi%3Dre%3Bslp_r_match%3D0%3Bsrpvid%3Db42595ebe16f00d7%3Bss_all%3D0%3Bssb%3Dempty%3Bsshis%3D0%26%3B&ss=Manila&is_ski_area=0&ssne=Manila&ssne_untouched=Manila&city=-2437894&checkin_year=2022&checkin_month=9&checkin_monthday=1&checkout_year=2022&checkout_month=9&checkout_monthday=2&group_adults=2&group_children=0&no_rooms=1&from_sf=1&nflt=price%3DPHP-10000-max-1%3Bht_id%3D204&shw_aparth=0%27'
search_html_2 = requests.get(url_10k_up, headers = headers)
search_soup_2 = BeautifulSoup(search_html_2.text, 'html.parser')
hotels_2 = search_soup_2.find_all('a', hotel_link_class)

for hotel in hotels_2 : # iteration to put hotel details in a dict for each hotel
    hotel_details_dict = {}
    # extract hotel name
    hotel_name = hotel.get('href').strip()[:-6].split('/')[-1].split('.')[0]
    title_hotel_name = hotel_name.replace('-',' ').title()
    hotel_details_dict['hotel_name'] = title_hotel_name
    # price range category based on website search
    hotel_details_dict['hotel_min_price_range_php'] = '10k and up'
    # extract reviews
    review_url = 'https://www.booking.com/reviews/ph/hotel/' + hotel_name + '.en-gb.html'
    review_html = requests.get(review_url, headers = headers)
    review_soup = BeautifulSoup(review_html.text, 'html.parser')
    reviewer_name = review_soup.find_all('p', {'class' : 'reviewer_name' } )
    review_content = review_soup.find_all('div', {'class' : 'review_item_review_content' } )
    zipped_reviews = dict(zip(reviewer_name, review_content) )
    review_count = 0
    for k, v in zipped_reviews.items() : # iterate to extract reviews with format
        review = 'Reviewer Name: {}\nReview: {}'.format(k.get_text(), v.get_text().strip().replace('\n', ' ') )
        review_count += 1
        if review_count == 3 : break
    hotel_details_dict['hotel_reviews'] = review 
    #extract hotel link
    hotel_details_dict['hotel_link'] = hotel.get('href').strip()[:-6]
    hotels_dict[hotel_name] = hotel_details_dict
    count += 1
    if count == 30 : break

# count number of hotels and print sample link
print('Scraping Complete!')
print('Count of Hotels in the List:', len(hotels_dict) )
print('Sample Hotel data:')
print(hotels_dict[hotel_name])<jupyter_output>Scraping Complete!
Count of Hotels in the List: 25
Sample Hotel data:
{'hotel_name': '<NAME>', 'hotel_min_price_range_php': '10k and up', 'hotel_reviews': 'Reviewer Name: \nJanet\n\nReview: No buffet yet and limited use of pool  Facilities is ok like indoor pool  Stayed in December 2021', 'hotel_link': 'https://www.booking.com/hotel/ph/marco-polo-ortigas-manila.en-gb.html?label=gen173nr-1DCAEoggI46AdIM1gEaLQBiAEBmAExuAEXyAEM2AED6AEB-AECiAIBqAIDuAK-vIuQBsACAdICJDNiYTIzMGMwLTgyYjUtNGFiYS04MjlkLTk1ZjhkMDZjZGM4YdgCBOACAQ&sid=c181dd9af9df88eff960c65192ff13a9&aid=304142&ucfs=1&arphpl=1&checkin=2022-09-01&checkout=2022-09-02&dest_id=-2437894&dest_type=city&group_adults=2&req_adults=2&no_rooms=1&group_children=0&req_children=0&hpos=10&hapos=10&sr_order=popularity&nflt=price%3DPHP-10000-max-1%3Bht_id%3D204&srpvid=98a795aeeaa001c4&srepoch=1645391838&all_sr_blocks=78158504_187848439_0_41_0&highlighted_blocks=78158504_187848439_0_41_0&matching_block_id=78158504_187848439_0_41_0&[...]<jupyter_text>Web scrape other hotel details from each hotel link and put in the hotels_dict (dictionary)<jupyter_code># scrape hotel other details from each hotel link in the dict and put in the hotels_dict (dictionary)
counter = 0
for k, v in hotels_dict.items() :
    hotel_link = v['hotel_link']
    # start with empty hotel details dict
    hotel_details_dict = {}
    
    # create soup object per hotel link
    hotel_html = requests.get(hotel_link, headers = headers)
    hotel_soup = BeautifulSoup(hotel_html.text, 'html.parser')
    
    # scrape and clean hotel location
    location_tag = hotel_soup.find('span',{'class' : 'hp_address_subtitle'} )
    hotel_loc = location_tag.contents[0].strip()
    hotels_dict[k]['hotel_location'] = hotel_loc 
    
    # scrape and clean amenities
    amenities_tag = hotel_soup.find_all('div', {'class' : 'important_facility'} )
    amenities_lst = []
    amenities = ''
    for amenity in amenities_tag:
        amenities_lst.append(amenity.get_text().strip() )
    
    amenities_lst = list(dict.fromkeys(amenities_lst) )
    for item in amenities_lst :
        if len(amenities) > 0 :
            amenities = amenities + ', ' + item
        else : 
            amenities = amenities + item
    
    hotels_dict[k]['hotel_amenities'] = amenities

    # scrape and clean no. of rooms
    room_types = hotel_soup.find_all('span', {'class' : 'hprt-roomtype-icon-link'} )
    count_room_avail = hotel_soup.find_all('span', {'class' : 'only_x_left urgency_message_red'} )
    room_types_lst = []
    avail_count_lst = []

    for room_type in room_types :
        room_types_lst.append((room_type.get_text().strip()) )
        
    if count_room_avail == [] :
        for count_room in range(len(room_types)) :
            avail_count = 'More than 7 available for this room type.'
            avail_count_lst.append(avail_count)
    else : 
        for count_room in count_room_avail :
            if len(count_room) > 0 :
                avail_count = count_room.get_text().strip()
                avail_count_lst.append(avail_count)
            else : 
                avail_count = 'More than 7 available for this room type.'
                avail_count_lst.append(avail_count)

    room_details = dict(zip(room_types_lst, avail_count_lst) )
    hotels_dict[k]['hotel_avail_rooms'] = room_details

    # scrape and clean other info
    other_tags = hotel_soup.find_all('div', {'id' : 'property_description_content'})
    for tag in other_tags :
        other_info = tag.get_text().lstrip().split('\n')
        other_info = other_info[0] + ' ' + other_info[1]
    
    hotels_dict[k]['hotel_other_info'] = other_info
    
    counter += 1
    # if counter == 1 : break # this is for testing purposes
    time.sleep(1) 

print('Scraping Complete!')
print('No. of iterations: ', counter)<jupyter_output>Scraping Complete!
No. of iterations:  25<jupyter_text>Create Pandas dataframe, clean, then save as Hotel.csv<jupyter_code>df = pd.DataFrame.from_dict(hotels_dict, orient = 'index') 
df.info()
df.head()
df['hotel_location'] = df['hotel_location'].apply(lambda x: x.split(',')[-3].strip() + ' City')
df.hotel_location.loc[df['hotel_location'] == '1588 Pedro Gil St. cor MH Del Pilar Malate City'] = 'Manila City'
df.hotel_location.loc[df['hotel_location'] == 'Muntinlupa City City'] = 'Muntinlupa City'
df.hotel_location.loc[df['hotel_location'] == 'Quezon City City'] = 'Quezon City'
df
df.drop('hotel_link', axis = 1)
df = df[['hotel_name', 'hotel_location', 'hotel_amenities', 'hotel_min_price_range_php', 
        'hotel_avail_rooms', 'hotel_other_info', 'hotel_reviews']]
df.to_csv('Hotel.csv', index = False)
df<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup