# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Exploring Environment Canada Weather Data with Python and Jupyter Notebooks
#
# - By <NAME> (<EMAIL>) - December 3rd 2017
#
# This notebook demonstrates how to download Environment Canada's weather data using Python with popular data analysis libraries like pandas and Beautiful Soup. Read the accompanying blog post [here](https://www.ubcenvision.com/blog/2017/11/30/jupyter-part1.html) on the [UBC Envision](https://www.ubcenvision.com) website for more details.
#
# To run this notebook on the cloud, use [Binder](https://mybinder.org/) or the [Syzygy](http://intro.syzygy.ca/getting-started/) platform. 
#
# Feel free to use, copy, paste, modify or distribute this notebook however you wish.
#
# *These examples are provided as-is with no guarantee that they will work in the future if Environment Canada modifies their API.*
#
# ## Updates:
# - March 13th 2020:
#     - Environment Canada API updated. `getHourlyData()` updated with skiprows=0.
#     - Updated requirements.txt so this notebook works on Binder without messing with the libraries
#
#
# - October 4th 2018: 
#     - Environment Canada API updated, we need to skip 15 rows instead of 16. `getHourlyData()` updated with skiprows=15

# # Part I: Data Extraction & Cleaning

import pandas as pd
import datetime
import matplotlib.pyplot as plt
import seaborn as sns
from dateutil import rrule
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import requests
import re

# +
# We'll need `fuzzywuzzy` to look up weather stations later
# Run "!pip install fuzzywuzzy --user" if you get an error

# # !pip install fuzzywuzzy --user
from fuzzywuzzy import fuzz
# -

# ## How to download one month of data:

# +
month = "01" # January
year = "2020" # 2020
stationID = 51442 #Vancouver

base_url = "http://climate.weather.gc.ca/climate_data/bulk_data_e.html?"
query_url = "format=csv&stationID={}&Year={}&Month={}&timeframe=1".format(stationID, year, month)
api_endpoint = base_url + query_url

print("Click me to download CSV data:")
print(api_endpoint)


# -

# ## Function for calling the Environment Canada API

# Call Environment Canada API
# Returns a dataframe of data
def getHourlyData(stationID, year, month):
    base_url = "http://climate.weather.gc.ca/climate_data/bulk_data_e.html?"
    query_url = "format=csv&stationID={}&Year={}&Month={}&timeframe=1".format(stationID, year, month)
    api_endpoint = base_url + query_url
    return pd.read_csv(api_endpoint, skiprows=0)


# ## How to download data between a specified date range

# +
stationID = 51442
start_date = datetime.strptime('Jun2015', '%b%Y')
end_date = datetime.strptime('Jun2016', '%b%Y')

frames = []
for dt in rrule.rrule(rrule.MONTHLY, dtstart=start_date, until=end_date):
    df = getHourlyData(stationID, dt.year, dt.month)
    frames.append(df)

weather_data = pd.concat(frames)
weather_data['Date/Time'] = pd.to_datetime(weather_data['Date/Time'])
weather_data['Temp (°C)'] = pd.to_numeric(weather_data['Temp (°C)'])
# -

# ## Plot average data and a rolling average
# Notice the broken lines, they indicate missing data points.

# %matplotlib inline
sns.set_style('whitegrid')
fig = plt.figure(figsize=(15,5))
plt.plot(weather_data['Date/Time'], weather_data['Temp (°C)'], '-o', alpha=0.8, markersize=2)
plt.plot(weather_data['Date/Time'], weather_data['Temp (°C)'].rolling(window=250,center=False).mean(), '-k', alpha=1.0)
plt.ylabel('Temp (°C)')
plt.xlabel('Time')
plt.show()

# ## Fix missing data points by interpolation

# Don't really care about accuracy right now, use simple linear interpolation
weather_data['Temp (°C)'] = weather_data['Temp (°C)'].interpolate()

# Then plot the data again:

# %matplotlib inline
sns.set_style('whitegrid')
fig = plt.figure(figsize=(15,5))
plt.plot(weather_data['Date/Time'], weather_data['Temp (°C)'], '-o', alpha=0.8, markersize=2)
plt.plot(weather_data['Date/Time'], weather_data['Temp (°C)'].rolling(window=250,center=False).mean(), '-k', alpha=1.0)
plt.ylabel('Temp (°C)')
plt.xlabel('Time')
plt.show()

# ## For convenience: Scrape StationIDs to lookup cities
# This section demonstrates a simple Python script for scraping StationIDs from Environment Canada using Beautiful Soup.
#
# The stationIDs are provided by province in this Environment Canada [page](http://climate.weather.gc.ca/historical_data/search_historic_data_e.html). Environment Canada limits the number of rows in the search results to 100 entries. This script loops through all pages and grabs the StationID, Station Name, Intervals and Year Range.
#

# +
# Specify Parameters
province = "BC"      # Which province to parse?
start_year = "2006"  # I want the results to go back to at least 2006 or earlier
max_pages = 5        # Number of maximum pages to parse, EC's limit is 100 rows per page, there are about 500 stations in BC with data going back to 2006

# Store each page in a list and parse them later
soup_frames = []

for i in range(max_pages):
    startRow = 1 + i*100
    print('Downloading Page: ', i)
    
    base_url = "http://climate.weather.gc.ca/historical_data/search_historic_data_stations_e.html?"
    queryProvince = "searchType=stnProv&timeframe=1&lstProvince={}&optLimit=yearRange&".format(province)
    queryYear = "StartYear={}&EndYear=2017&Year=2017&Month=5&Day=29&selRowPerPage=100&txtCentralLatMin=0&txtCentralLatSec=0&txtCentralLongMin=0&txtCentralLongSec=0&".format(start_year)
    queryStartRow = "startRow={}".format(startRow)

    response = requests.get(base_url + queryProvince + queryYear + queryStartRow) # Using requests to read the HTML source
    soup = BeautifulSoup(response.text, 'html.parser') # Parse with Beautiful Soup
    soup_frames.append(soup)
# -

# ## Parsing the Environment Canada page with Beautiful Soup

# +
# Empty list to store the station data
station_data = []

for soup in soup_frames: # For each soup
    forms = soup.findAll("form", {"id" : re.compile('stnRequest*')}) # We find the forms with the stnRequest* ID using regex 
    for form in forms:
        try:
            # The stationID is a child of the form
            station = form.find("input", {"name" : "StationID"})['value']
            
            # The station name is a sibling of the input element named lstProvince
            name = form.find("input", {"name" : "lstProvince"}).find_next_siblings("div")[0].text
            
            # The intervals are listed as children in a 'select' tag named timeframe
            timeframes = form.find("select", {"name" : "timeframe"}).findChildren()
            intervals =[t.text for t in timeframes]
            
            # We can find the min and max year of this station using the first and last child
            years = form.find("select", {"name" : "Year"}).findChildren()            
            min_year = years[0].text
            max_year = years[-1].text
            
            # Store the data in an array
            data = [station, name, intervals, min_year, max_year]
            station_data.append(data)
        except:
            pass

# Create a pandas dataframe using the collected data and give it the appropriate column names
stations_df = pd.DataFrame(station_data, columns=['StationID', 'Name', 'Intervals', 'Year Start', 'Year End'])
stations_df.head()
# -

# ## Filtering Station Data
# For example, to show only stations with hourly data, use the map() function.

# Show only data with hourly intervals
hourly_stations = stations_df.loc[stations_df['Intervals'].map(lambda x: 'Hourly' in x)]
hourly_stations.head()

# ## Looking up stations
# Now that we have the stationIDs, we can use Fuzzywuzzy and fuzzy string matching to lookup the station IDs with the city names. Here are 2 examples

# +
# Find the stations that are in Whistler
string = "Whistler"
tolerance = 90

hourly_stations[hourly_stations['Name'].apply(lambda x: fuzz.token_set_ratio(x, string)) > tolerance]

# +
# Find the stations that are in Vancouver
string = "Vancouver"
tolerance = 90

hourly_stations[hourly_stations['Name'].apply(lambda x: fuzz.token_set_ratio(x, string)) > tolerance]
# -

# ## Download Weather Data
# Now that we know the station ID, we can use our getHourlyData() function to call the Environment Canada API with the station ID that we found. 
#
# Here's how we get Whistler data from November 2016 to November 2017:

# +
# Get Whistler weather data for November 2016 to November 2017
stationID = 43443
start_date = datetime.strptime('Nov2016', '%b%Y')
end_date = datetime.strptime('Nov2017', '%b%Y')

frames = []
for dt in rrule.rrule(rrule.MONTHLY, dtstart=start_date, until=end_date):
    df = getHourlyData(stationID, dt.year, dt.month)
    frames.append(df)

whistler = pd.concat(frames)
whistler['Date/Time'] = pd.to_datetime(whistler['Date/Time'])
whistler['Temp (°C)'] = pd.to_numeric(whistler['Temp (°C)'])
whistler.head()
# -

# ## Let's plot the data for Whistler

# %matplotlib inline
sns.set_style('whitegrid')
fig = plt.figure(figsize=(15,5))
plt.plot(whistler['Date/Time'], whistler['Temp (°C)'], '-o', alpha=0.8, markersize=2)
plt.plot(whistler['Date/Time'], whistler['Temp (°C)'].rolling(window=250,center=False).mean(), '-k', alpha=1.0)
plt.ylabel('Temp (°C)')
plt.xlabel('Time')
plt.show()

# ## How to fix missing data points with interpolation

# +
# Find the number of rows with a 'M' for missing temperature flag, or NaN for the actual temperature value
print('Missing data rows: ', whistler.loc[(~whistler['Temp Flag'].isnull()) | (whistler['Temp (°C)'].isnull())].shape[0])

# Do interpolation 
whistler['Temp (°C)'] = whistler['Temp (°C)'].interpolate()

# Did we fix everything?
print('Missing data rows: ', whistler.loc[(whistler['Temp (°C)'].isnull())].shape[0])
# -

# Re-plot the data

# %matplotlib inline
sns.set_style('whitegrid')
fig = plt.figure(figsize=(15,5))
plt.plot(whistler['Date/Time'], whistler['Temp (°C)'], '-o', alpha=0.8, markersize=2)
plt.plot(whistler['Date/Time'], whistler['Temp (°C)'].rolling(window=250,center=False).mean(), '-k', alpha=1.0)
plt.ylabel('Temp (°C)')
plt.xlabel('Time')
plt.show()

# # Exporting Data

# We'll export the dataframes in CSV format so we don't have to re-download the data every time we restart Jupyter:

stations_df.to_csv('stations.csv')
whistler.to_csv('whistler.csv')

StarcoderdataJupyterScriptsDedupFiltered