# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

from bs4 import BeautifulSoup
from splinter import Browser
import pandas as pd
import datetime as dt
import requests
import json

# +
urls_lost = []
urls_found = []
urls_reunited = []

city = "oakland"
state = "ca"
page_no = 1


"""
city = input("Please type a city: ")
state = input("Please type the 2-letter state abbreviation: ")
"""

# status codes: 100 = lost; 101 = found; 102 = reunited

for status in range(100,103):
    url = 'https://www.pawboost.com/lost-found-pets/' + \
        city + '-' + state + '-/all-lost-found-stray-reunited-cats/page-' + str(page_no) + \
        '?LfdbFeedStatusForm%5Bstatus%5D%5B%5D=' + str(status) + '&LfdbFeedStatusForm%5Baddress%5D=' + \
        city + '%2C%20' + state + \
        '&LfdbFeedStatusForm%5Bradius%5D=15&LfdbFeedStatusForm%5Bspecies%5D=Cat' + \
        '&LfdbFeedStatusForm%5BsortAttribute%5D=distance&LfdbFeedStatusForm%5BdateRange%5D=90'


    response = requests.get(url)
    html = response.content

    soup = BeautifulSoup(html, "html.parser")
    soup.prettify()

    if status == 100:
        print("Lost cats:")
    elif status == 101:
        print("Found cats:")
    elif status == 102:
        print("Reunited w/ family:")
    else:
        print("error")
    
    print(url)
    print("URLs found:")
    
    for anchor in soup.findAll('a', attrs={'class': 'btn btn-primary width-full'}, href=True):
        print(anchor['href'])
        if status == 100:
            urls_lost.append(anchor['href'])
        elif status == 101:
            urls_found.append(anchor['href'])
        elif status == 102:
            urls_reunited.append(anchor['href'])
    
# -

for status in range(100,103):
    if status == 100:
        length = len(urls_lost)
        active_list = urls_lost
    elif status == 101:
        length = len(urls_found)
        active_list = urls_found
    elif status == 102:
        length = len(urls_reunited)
        active_list = urls_reunited
    
    pet_url = []
    pet_id = []
    pet_name = []
    pet_gender = []
    pet_status = []
    pet_description = []
    pet_address = []
    pet_address_clean = []
    
    # first url is irrelevant due to featured pet
    for i in range(1,length):
        new_url = active_list[i]
        pet_url.append(new_url)
        
        response = requests.get(new_url)
        html = response.content

        soup = BeautifulSoup(html, "html.parser")
        soup.prettify()
        # print(response)

        item_no = []
        h2_tags = []
        p_tags = []

        for i in range(0, 12):
            paragraphs = soup.find_all("h2")[i].text
            item_no.append(i)
            h2_tags.append(paragraphs)
    
        for i in range(0, 20):
            paragraphs = soup.find_all("p")[i].text
            item_no.append(i)
            p_tags.append(paragraphs)

        # for viewing which text fields are relevant
        results_h2 = pd.DataFrame({"text": h2_tags})
        results_p = pd.DataFrame({"text": p_tags})

        # pet ID
        pet_id.append(results_h2['text'][3]) 
        
        # pet name
        if status == 100:
            pet_name.append(results_h2['text'][4])
        elif status == 101:
            pet_name.append("Unknown")
        elif status == 102:
            pet_name.append(results_h2['text'][4])
        
        # gender
        if status == 100:
            pet_gender.append(results_h2['text'][5])
        elif status == 101:
            pet_gender.append(results_h2['text'][4])
        elif status == 102:
            pet_gender.append(results_h2['text'][5])
        
        # status
        if status == 100:
            pet_status.append(results_p['text'][4])
        elif status == 101:
            pet_status.append(results_p['text'][3])
        elif status == 102:
            pet_status.append(results_p['text'][4])
            
        # description
        if status == 100:
            pet_description.append(results_p['text'][10])
        elif status == 101:
            pet_description.append(results_p['text'][9])
        elif status == 102:
            pet_description.append(results_p['text'][8])
        
        # address, if available
        if status == 100:
            address = results_h2['text'][8] + " " + results_h2['text'][7]
        elif status == 101:
            address = results_h2['text'][7] + " " + results_h2['text'][6]
        elif status == 102:
            address = results_h2['text'][7]
               
        pet_address.append(address)
        
        address_clean = address.replace(' ', '+').replace('&', 'and')
        pet_address_clean.append(address_clean)
        # print(address_clean)
        
        if status == 100:
            details_lost = pd.DataFrame({"url": pet_url,
                "ref_no": pet_id,
                "pet_name": pet_name,
                "gender": pet_gender,
                "status": pet_status,
                "description": pet_description,
                "address": pet_address,
                "address_clean": pet_address_clean})
        elif status == 101:
            details_found = pd.DataFrame({"url": pet_url,
                "ref_no": pet_id,
                "pet_name": pet_name,
                "gender": pet_gender,
                "status": pet_status,
                "description": pet_description,
                "address": pet_address,
                "address_clean": pet_address_clean})
        elif status == 102:
            details_reunited = pd.DataFrame({"url": pet_url,
                "ref_no": pet_id,
                "pet_name": pet_name,
                "gender": pet_gender,
                "status": pet_status,
                "description": pet_description,
                "address": pet_address,
                "address_clean": pet_address_clean})
    

details_lost

details_found

details_reunited

# +
# need to do api call because coordinates could not be scraped from page
for status in range(100,103):
    if status == 100:
        length = len(urls_lost)
    elif status == 101:
        length = len(urls_found)
    elif status == 102:
        length = len(urls_reunited)
        
    latitude = []
    longitude = []
    
    for i in range(0,length-1):
        if status == 100:
            query_address = details_lost['address_clean'][i]
        elif status == 101:
            query_address = details_found['address_clean'][i]
        elif status == 102:
            query_address = details_reunited['address_clean'][i]
        
        parameters = query_address
        api_key = #REDACTED

        query_url = "https://maps.googleapis.com/maps/api/geocode/json?address=" + \
            parameters + "&key=" + api_key

        print(f"finding coordinates for: {query_address}")
        #print(query_url)
        response = requests.get(query_url)
        gmap_json = response.json()

        try:
            print(gmap_json['results'][0]['geometry']['location']['lat'])
            latitude.append(float(gmap_json['results'][0]['geometry']['location']['lat']))
        except (KeyError, IndexError):
            print("could not find latitude")
            latitude.append(None)      
        try:
            print(gmap_json['results'][0]['geometry']['location']['lng'])
            longitude.append(float(gmap_json['results'][0]['geometry']['location']['lng']))
        except (KeyError, IndexError):
            print("could not find longitude")
            longitude.append(None)
            
        print("--------------------------")
        
    if status == 100:
        details_lost['latitude'] = latitude
        details_lost['longitude'] = longitude
    elif status == 101:
        details_found['latitude'] = latitude
        details_found['longitude'] = longitude
    elif status == 102:
        details_reunited['latitude'] = latitude
        details_reunited['longitude'] = longitude

print("Coordinate lookup completed")               
# -

details_lost

details_found

details_reunited

details_lost.to_csv("details_lost.csv", index=True, header=True)
details_found.to_csv("details_found.csv", index=True, header=True)
details_reunited.to_csv("details_reunited.csv", index=True, header=True)



StarcoderdataJupyterScriptsDedupFiltered