<jupyter_start><jupyter_text>Entropy and Persistent Homology &copy;  [<NAME>](<EMAIL>)  Industrial and Enterprise Systems Engineering, The Grainger College of Engineering,  UIUC   [Reference](https://towardsdatascience.com/how-to-pull-data-from-an-api-using-python-requests-edcc8d6441b1) Part 1: Download Data from INaturalist website using APIFrom API documentation,we get the following information:`Please note that we throttle API usage to a max of 100 requests per minute, though we ask that you try to keep it to 60 requests per minute or lower, and to keep under 10,000 requests per day. If we notice usage that has serious impact on our performance we may institute blocks without notification.``per_pageAllowed values: 1 to 200`Locations data collected from:* place_id: 1563* place_id: 49906<jupyter_code># importing required libraries
import requests
import math
import time
import pandas as pd
from datetime import datetime
import os
# Data for storing all the pulled data
data = {
    'time_observed_at': list(),
    'species_guess': list(),
    'genus_name': list(),
    'rank': list(),
    'wikipedia_url': list(),
    'iconic_taxon_name': list(),
    'preferred_common_name': list(),
    'uri': list(),
    'longitude': list(),
    'latitude': list(),
    'place_guess': list()
}<jupyter_output><empty_output><jupyter_text>Part 1.1. Function for getting the dictionary of pages and page_numbersFor each page, there is a different per_page limit. This was not mentioned in the api documentation of the inaturalist website.<jupyter_code># pages = {}
# place_id = 1563
# page_number = 1
# per_page = 200 #max_limit
# i = 1
# sleep_time = 60
# while True:
#   while per_page != 0:
#     #go through each page and per_page, get the request's response, if it is 200, append page_number:per_page to the dictionary.
#     #if for any per_page, we get status_code != 200, we decrement the per_page by 1, and check response. 
    
#     response = requests.get("https://api.inaturalist.org/v1/observations?place_id={}&page={}&per_page={}".format(place_id, page_number, per_page))
#     if i%50 == 0: #We have a limit of 60 pages per minute, for pulling data from API
#         time.sleep(sleep_time)
#     if response.status_code == 200:
#       pages[page_number] = per_page
#       page_number +=1
#     elif response.status_code != 200:
#       per_page -= 1
#     i += 1
#   break 

def findPerPage(place_id, page_number, per_page, how_many_pages):
    pages = {}
    i = 1
    sleep_time = 60
    while True:
        while per_page != 0:
            #go through each page and per_page, get the request's response, 
            # if it is 200, append page_number:per_page to the dictionary.
            # if for any per_page, we get status_code != 200, we decrement the per_page by 1, and check response. 
            response = requests.get("https://api.inaturalist.org/v1/observations?place_id={}&page={}&per_page={}".format(place_id, page_number, per_page))
            if i%50 == 0: #We have a limit of 60 pages per minute, for pulling data from API
                time.sleep(sleep_time)
            if response.status_code == 200:
                pages[page_number] = per_page
                page_number +=1
            elif response.status_code != 200:
                per_page -= 1
            i += 1

            # if how_many_pages >= 200:
            #     return pages
        break
    return pages

def download_csv_pages(pages):
    pages_Series = pd.Series(pages)
    #pages_Series.to_csv('pages.csv')
    pages_df = pd.DataFrame(pages_Series, columns=['page_number', 'per_page'])
    pages_df.to_csv('pages.csv')

def download_pickle_pages(pages):
    pages_Series = pd.Series(pages)
    #pages_Series.to_csv('pages.csv')
    pages_df = pd.DataFrame(pages_Series, columns=['page_number', 'per_page'])
    pages_df.to_pickle('pages.pkl')
# pages_Series = pd.Series(pages)
# pages_Series.to_csv('pages.csv')
# pages_df = pd.read_csv('pages.csv', names=['page_number', 'per_page'])
# pages_df.info()
# pages_df.to_csv('pages.csv')
# pages_df.to_pickle('pages.pkl')<jupyter_output><empty_output><jupyter_text>Part 1.2. Pull data<jupyter_code># Function for pulling the data
def pull_data(data, place_id, sleep_time, page_number, per_page):
    '''
    place_id: each location has a place_id, which we can get from the website
    sleep_time: time in seconds we want to sleep
    page_number: depends on the number of observations. total_observations/per_page
    per_page: #observations per page
    '''
    if page_number%40 == 0:
        time.sleep(sleep_time)
    else:
        response = requests.get("https://api.inaturalist.org/v1/observations?place_id={}&page={}&per_page={}".format(place_id, page_number, per_page))
        file_dict = response.json()
        if ('results' in file_dict):
            for j in range(per_page):
            
                if 'time_observed_at' in file_dict['results'][j]:
                    data['time_observed_at'].append(file_dict['results'][j]['time_observed_at'])
                else:
                    data['time_observed_at'].append(None)
                if 'species_guess' in file_dict['results'][j]:
                    data['species_guess'].append(file_dict['results'][j]['species_guess'])
                else:
                    data['species_guess'].append(None)

                if ('taxon' in file_dict['results'][j]) & (file_dict['results'][j]['taxon'] is not None) :
                    #print(j)
                    if 'name' in file_dict['results'][j]['taxon']:
                        data['genus_name'].append(file_dict['results'][j]['taxon']['name'])
                    else:
                        data['genus_name'].append(None)
                    if 'rank' in file_dict['results'][j]['taxon']:
                        data['rank'].append(file_dict['results'][j]['taxon']['rank'])
                    else:
                        data['rank'].append(None)
                    if 'wikipedia_url' in file_dict['results'][j]['taxon']:
                        data['wikipedia_url'].append(file_dict['results'][j]['taxon']['wikipedia_url'])
                    else:
                        data['wikipedia_url'].append(None)
                    if 'iconic_taxon_name' in file_dict['results'][j]['taxon']:
                        data['iconic_taxon_name'].append(file_dict['results'][j]['taxon']['iconic_taxon_name'])
                    else:
                        data['iconic_taxon_name'].append(None)
                    #print(j)
                    if 'preferred_common_name' in file_dict['results'][j]['taxon']:
                        data['preferred_common_name'].append(file_dict['results'][j]['taxon']['preferred_common_name'])
                    else:
                        data['preferred_common_name'].append(None)
                else:
                    data['genus_name'].append(None)
                    data['rank'].append(None)
                    data['wikipedia_url'].append(None)
                    data['iconic_taxon_name'].append(None)
                    data['preferred_common_name'].append(None)
                if 'uri' in file_dict['results'][j]:
                    data['uri'].append(file_dict['results'][j]['uri'])
                else:
                    data['uri'].append(None)
                if 'geojson' in file_dict['results'][j]:
                    data['longitude'].append(file_dict['results'][j]['geojson']['coordinates'][0])
                    data['latitude'].append(file_dict['results'][j]['geojson']['coordinates'][1])
                else:
                    data['longitude'].append(None)
                    data['latitude'].append(None)
                if 'place_guess' in file_dict['results'][j]:
                    data['place_guess'].append(file_dict['results'][j]['place_guess'])
                else:
                    data['place_guess'].append(None)
        else:
            print(page_number)<jupyter_output><empty_output><jupyter_text>`pages.csv' contains the ```python{'page_number': 'per_page'}```dictionary. For each page, we have a maximum limit on the number of observations that one can pull.<jupyter_code>pages = pd.read_csv('pages.csv')
# Uncomment for pulling data. Change place_id
# for i in range(1, len(pages)):

#     pull_data(data, 1563, 60, int(pages['page_number'][i]), int(pages['per_page'][i]))
# data_df = pd.DataFrame(data)
# data_df.info()

# place_id = 1563
# csv_file = 'data_' + str(place_id) + '_' + str(datetime.now().strftime('%Y_%m_%d_%H_%M_%S')) + '.csv'
# pickle_file = 'data_' + str(place_id) + '_' + str(datetime.now().strftime('%Y_%m_%d_%H_%M_%S')) + '.pkl'
# data_df.to_csv(csv_file)
# data_df.to_pickle(pickle_file)
# data_df.info()

def getData(pages_df, data, place_id):

    for i in range(1, len(pages)):
        pull_data(data, place_id, 60, int(pages_df['page_number'][i]), int(pages_df['per_page'][i]))
    data_df = pd.DataFrame(data)
    return data_df
def download_csv_data(data, place_id, ):

    csv_file = 'data_' + str(place_id) + '_' + str(datetime.now().strftime('%Y_%m_%d_%H_%M_%S')) + '.csv'
    pickle_file = 'data_' + str(place_id) + '_' + str(datetime.now().strftime('%Y_%m_%d_%H_%M_%S')) + '.pkl'
    data_df.to_csv(csv_file)
    data_df.to_pickle(pickle_file)<jupyter_output><empty_output><jupyter_text>***Use pickle instead of csv***<jupyter_code>data_df = pd.read_csv('data_1563_2021_06_24_12_57_28.csv', index_col= 0)
# data_df.to_pickle('data_1563_2021_06_06_20_47_20.pickle')
# data_df = pd.read_pickle('data_1563_2021_06_06_20_47_20.pickle')
data_df.info()
data_df.dropna(subset = ['wikipedia_url'], inplace = True)
data_df.info()
data_df.reset_index(inplace=True, drop = True)
data_df.head(15)<jupyter_output><empty_output><jupyter_text>Part 2. Get taxonomy data from the rows Search for a term in wikipedia search and get its pageUncomment for looking at results. I didn't find it useful.<jupyter_code># import wikipedia
# result = wikipedia.search("Monarda_fistulosa")
# print(result)
# # get the page: Neural network
# page = wikipedia.page(result[0])
# print(page)
# # get the title of the page
# title = page.title
# print(title)
# # get the categories of the page
# categories = page.categories
# print(categories)
# # get the whole wikipedia page text (content)
# content = page.content
# print(content)
# # get all the links in the page
# links = page.links
# print(links)
# # get the page references
# references = page.references
# print(references)
# # summary
# summary = page.summary
# print(summary)<jupyter_output><empty_output><jupyter_text>Part 2.1. Scraping data using beautiful soup<jupyter_code># For each row, if wikipedia_uri exists, we will use beautiful soup to extract the data 
# related to taxonomy
# import required modules
import requests
from bs4 import BeautifulSoup
# get URL
page = requests.get("http://en.wikipedia.org/wiki/Monarda_fistulosa")
  
# display status code
# print(page.status_code)
  
# display scrapped data
# print(page.content)

# scrape webpage
soup = BeautifulSoup(page.content, 'html.parser') #.get_text(strip=True) #This removes \xa0

# display scrapped data
# print(soup.prettify())

# list(soup.children)

# find all occurance of p in HTML
# includes HTML tags
# print(soup.find_all('p'))
data_df[data_df['rank'] == 'species' ]['wikipedia_url'][5]<jupyter_output><empty_output><jupyter_text>Part 3.1.1. Get table from the id<jupyter_code># create object
# object = soup.find(id="mw-content-text")

# # find tags
# items = object.find_all(class_="infobox biota")
# result = items[0]
  
# # display tags
# print(result.prettify())

table = soup.find_all('table')

# table[0]

# for child in soup.find_all('table')[0].children:
#     for td in child:
#         print(td)

# list(soup.find_all('table')[0].tr.next_siblings)

# for sibling in soup.find_all('table')[0].tr.next_siblings:
#     for td in sibling:
#         print(td)

table = soup.find('table', attrs={'class':'infobox biota'}) #class="infobox biota"
table_rows = table.find_all('tr')

# table_rows

# len(table_rows)
data_taxonomy = {
    'Kingdom': list(),
    'Phylum': list(),
    'Class': list(),
    'Order': list(),
    'Suborder': list(),
    'Family': list(),
    'Genus': list(),
    'Species': list()
}
l = []
data_taxonomy['Kingdom'].insert(0, None)
data_taxonomy['Phylum'].insert(0, None)
data_taxonomy['Class'].insert(0, None)
data_taxonomy['Order'].insert(0, None)
data_taxonomy['Suborder'].insert(0, None)
data_taxonomy['Family'].insert(0, None)
data_taxonomy['Genus'].insert(0, None)
data_taxonomy['Species'].insert(0, None)
for tr in table_rows:
    td = tr.find_all('td')
    row = [tr.text.replace('\n', '').replace(':', '').replace(u'\xa0', ' ') for tr in td]
    

    if 'Kingdom' in row:
        data_taxonomy['Kingdom'][0] = row[1]
        l.append(row)
    if 'Phylum' in row:
        data_taxonomy['Phylum'][0] = row[1]
        l.append(row)   
    if 'Class' in row:
        data_taxonomy['Class'][0] = row[1]
        l.append(row)
    if 'Order' in row:
        data_taxonomy['Order'][0] = row[1]
        l.append(row)
    if 'Suborder' in row:
        data_taxonomy['Suborder'][0] = row[1]
        l.append(row)
    if 'Family' in row:
        data_taxonomy['Family'][0] = row[1]
        l.append(row)
    if 'Genus' in row:
        data_taxonomy['Genus'][0] = row[1]
        l.append(row)
    if 'Species' in row:
        data_taxonomy['Species'][0] = row[1]
        l.append(row)
    
    # elif 'Clade' in row:
    #     l.append(row)
l

# 'Kingdom' in l[4]

# table_rows = table_rows[4:13]

# l = []
# for tr in table_rows:
#     td = tr.find_all('td')
#     row = [tr.text for tr in td]
#     l.append(row)

# l
data_taxonomy
len(data_df)
data_taxonomy = {
    'Kingdom': list(),
    'Phylum': list(),
    'Class': list(),
    'Order': list(),
    'Suborder': list(),
    'Family': list(),
    'Genus': list(),
    'Species': list()
}
for i in range(len(data_df)):
    if 'http' in data_df['wikipedia_url'][i]:
        #print(i)
        # get URL
        data_taxonomy['Kingdom'].insert(i, None)
        data_taxonomy['Phylum'].insert(i, None)
        data_taxonomy['Class'].insert(i, None)
        data_taxonomy['Order'].insert(i, None)
        data_taxonomy['Suborder'].insert(i, None)
        data_taxonomy['Family'].insert(i, None)
        data_taxonomy['Genus'].insert(i, None)
        data_taxonomy['Species'].insert(i, None)

        page = requests.get(data_df['wikipedia_url'][i])

        # scrape webpage
        soup = BeautifulSoup(page.content, 'html.parser')

        #table = soup.find_all('table')

        table = soup.find('table', attrs={'class':'infobox biota'}) #class="infobox biota"
        if table is not None:
        
        
            table_rows = table.find_all('tr')
            for tr in table_rows:
                td = tr.find_all('td')
                row = [tr.text.replace('\n', '').replace(':', '').replace(u'\xa0', ' ') for tr in td]
        

                if 'Kingdom' in row:
                    data_taxonomy['Kingdom'][i] = row[1]
                    #l.append(row)
                if 'Phylum' in row:
                    data_taxonomy['Phylum'][i] = row[1]
                    #l.append(row)   
                if 'Class' in row:
                    data_taxonomy['Class'][i] = row[1]
                    #l.append(row)
                if 'Order' in row:
                    data_taxonomy['Order'][i] = row[1]
                    #l.append(row)
                if 'Suborder' in row:
                    data_taxonomy['Suborder'][i] = row[1]
                    #l.append(row)
                if 'Family' in row:
                    data_taxonomy['Family'][i] = row[1]
                    #l.append(row)
                if 'Genus' in row:
                    data_taxonomy['Genus'][i] = row[1]
                    #l.append(row)
                if 'Species' in row:
                    data_taxonomy['Species'][i] = row[1]
                    #l.append(row)
    

len(data_taxonomy['Species'])
data_taxonomy_df = pd.DataFrame(data_taxonomy) 
data_taxonomy_df.to_csv('data_taxonomy_df.csv')
page = requests.get(data_df['wikipedia_url'][0])
data_taxonomy_df.head()<jupyter_output><empty_output>