# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: PythonData
#     language: python
#     name: pythondata
# ---

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import session
from sqlalchemy import create_engine, func
from config import db_password
import psycopg2
from sklearn.preprocessing import StandardScaler
from datetime import date, datetime
from path import Path
from sklearn import tree
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report


# ## Import Data

# +
Base = automap_base()
engine = create_engine(f"postgres://caznoe:{db_password}@aves.cos0wnwxlodh.us-east-2.rds.amazonaws.com:5432/aves")

Base.prepare(engine, reflect=True)
# print(engine.table_names())

# +
bird_df = pd.read_sql("SELECT * FROM bird_aqi_weather", engine)

bird_df.head()

# -

pop_df = pd.read_csv('population.csv')  
pop_df.tail()

# +
# pop_df.info()
# -

# ## Cleaning

# Make year column into datetime
year_column = pd.to_datetime(bird_df['year'], infer_datetime_format=True)

# +
# Bin dates into seasons 

Y = 2000 # dummy leap year to allow input X-02-29 (leap day)
seasons = [('winter', (date(Y,  1,  1),  date(Y,  3, 20))),
           ('spring', (date(Y,  3, 21),  date(Y,  6, 20))),
           ('summer', (date(Y,  6, 21),  date(Y,  9, 22))),
           ('autumn', (date(Y,  9, 23),  date(Y, 12, 20))),
           ('winter', (date(Y, 12, 21),  date(Y, 12, 31)))]

def get_season(now):
    if isinstance(now, datetime):
        now = now.date()
    now = now.replace(year=Y)
    return next(season for season, (start, end) in seasons
                if start <= now <= end)

date_column = pd.to_datetime(bird_df['date'], infer_datetime_format=True).map(lambda x : get_season(x))
bird_df['date']= date_column


# +
# Split Year column into Year, Month, Day to prepare or merge
bird_df['Year']=[d.split('-')[0] for d in bird_df.year]
bird_df['Month']=[d.split('-')[1] for d in bird_df.year]
bird_df['Day']=[d.split('-')[2] for d in bird_df.year]

bird_df.head()

# +
# bird_df.info()
# -

# Drop Month, year, and Day
new_df= bird_df.drop(["Month", "year","Day"], axis=1)
new_df.head()

# Make "Year" column numeric
new_df= new_df.astype({'Year': 'int64'})


# +
# Renaming Columns for merge prep
new_df = new_df.rename(columns={'county_name': 'County', 'Year':'Year', 'date': 'Season', 'aqi':'AQI', 'category':'Category'})

new_df.head()
# -

# Merge population data and ebird/air quality data
newer_df = pd.merge(new_df, pop_df, on=['County','Year'])
newer_df.head()

# Replace "None" with 0
newer_df.replace(to_replace=[None], value="0", inplace=True)
newer_df.head()

# +
#newer_df.to_csv("birdAQIPop.csv")
# -

# Encode hummingbird Species
species_num = {
   "Calypte anna": 1,
   "Selasphorus sasin": 2,
   "Calypte costae": 3,
   "Selasphorus rufus": 4,
   "Archilochus alexandri": 5,
   "Selasphorus calliope": 6,
   "Cynanthus latirostris": 7,
   "Selasphorus platycercus": 8,
   "Archilochus colubris": 9,
   "Amazilia violiceps": 10
}

# Create new column with encoded species
newer_df["species_num"] = newer_df["species"].apply(lambda x: species_num[x])


# Drop old unencoded species column
newer_df = newer_df.drop(["species"], axis=1)


newest_df= newer_df.copy()
newest_df.head()

# Change columns from object to numeric value
newest_df['heavy_rain'] = pd.to_numeric(newest_df['heavy_rain'])
newest_df['high_wind'] = pd.to_numeric(newest_df['high_wind'])


counties_num ={'Kern':1 , 'San Diego':2, 'Marin':3, 'Santa Cruz':4, 'Mendocino':5, 'Fresno':6,
       'Orange':7, 'Napa':8, 'San Joaquin':9, 'Alameda':10, 'Ventura':11, 'Tulare':12,
       'San Mateo':13, 'Solano':14, 'Monterey':15, 'Sonoma':16, 'Santa Clara':17,
       'Humboldt':18, 'Lake':19, 'Riverside':20, 'Madera':21, 'Stanislaus':22, 'Yolo':23,
       'Siskiyou':24, 'Amador':25, 'Butte':26, 'Sacramento':27, 'Los Angeles':28,
       'Santa Barbara':29, 'San Francisco':30, 'San Bernardino':31, 'Plumas':32,
       'El Dorado':33, 'Calaveras':34, 'Placer':35, 'Mariposa':36, 'Tehama':37,
       'Imperial':38, 'Merced':39, 'Contra Costa':40, 'Mono':41, 'Inyo':42, 'Tuolumne':43,
       'Kings':44, 'Nevada':45, 'Shasta':46, 'Sutter':47, 'San Benito':48, 'Del Norte':49,
       'San Luis Obispo':50, 'Glenn':51, 'Colusa':52, 'Trinity':53, 'Alpine':54}

# Create new column with encoded counties
newest_df["counties_num"] = newest_df["County"].apply(lambda x: counties_num[x])
newest_df.head()

newest_df= newest_df.drop(["County"], axis=1)

# +
# Create new column with encoded Defining parameter

parameter_num={'Ozone':1, 'PM2.5':2, 'PM10':3, 'CO':4, 'NO2':5, 'SO2':5}
# -

newest_df["parameter_num"] = newest_df["defining_parameter"].apply(lambda x: parameter_num[x])
newest_df= newest_df.drop(["defining_parameter"], axis=1)

# +
# Create new column with encoded seasons

season_num= {'spring':1, 'autumn':2, 'winter':3, 'summer':4}
# -

newest_df["season_num"] = newest_df["Season"].apply(lambda x: season_num[x])
newest_df= newest_df.drop(["Season"], axis=1)

newest_df.head()

# +
# Make population a numeric datatype 
newest_df['Population'] = newest_df['Population'].str.replace(',', '').astype(float)

newest_df['Population'] = pd.to_numeric(newest_df['Population'])

# -

# Make Heavy raind and high wind into numeric int
newest_df['heavy_rain'] = pd.to_numeric(newest_df['heavy_rain'])
newest_df['high_wind'] = pd.to_numeric(newest_df['high_wind'])

#Drop category
newest_df = newest_df.drop(["Category"], axis=1)


newest_df.info()

# +
# newest_df.to_csv("encoded_Bird.csv")
# -

list(newest_df.columns.values)

# # Stats Summary

# +
import seaborn as sn
import matplotlib.pyplot as plt

df_corr = pd.DataFrame(newest_df, columns= ['AQI',
 'heavy_rain',
 'high_wind',
 'Year',
 'Population',
 'species_num',
 'counties_num',
 'category_num',
 'parameter_num',
 'season_num'])

corrMatrix = df_corr.corr()
sn.heatmap(corrMatrix, annot=True)
plt.savefig("corrMatrix.png")

plt.show()
# -

# # Create training and test sets

# +
# Remove bird observation outcome target from features data
y = newest_df.species_num
X = newest_df.drop(columns="species_num")

# Split training/test datasets
# X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, stratify=y)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
# -

# # Apply scaling

# +
# from sklearn.preprocessing import MinMaxScaler
# scaler = MinMaxScaler()
# X_train_scaled = scaler.fit_transform(X_train)
# X_test_scaled = scaler.transform(X_test)

# Scale 
# Creating a StandardScaler instance.
scaler = StandardScaler()
# Fitting the Standard Scaler with the training data.
X_scaler = scaler.fit(X_train)

# Scaling the data.
X_train_scaled = X_scaler.transform(X_train)
X_test_scaled = X_scaler.transform(X_test)
# -

# # Build Models 

# +
## Logistic Regression
from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
logreg.fit(X_train_scaled, y_train)

print('Accuracy of logistic regression on training', logreg.score(X_train_scaled, y_train))
print('Accuracy of logistic regression on testing', logreg.score(X_test_scaled, y_test))

# +
## Decision Tree
from sklearn.tree import DecisionTreeClassifier
dt = DecisionTreeClassifier()
dt.fit(X_train_scaled, y_train)

print('Accuracy of Decision tree on training', dt.score(X_train_scaled, y_train))
print('Accuracy of Decision tree on testing', dt.score(X_test_scaled, y_test))
# -

# Setting max decision tree depth to help avoid overfitting
dt2 = DecisionTreeClassifier(max_depth=3)
dt2.fit(X_train_scaled, y_train)
print('Accuracy of Decision tree on training', dt2.score(X_train_scaled, y_train))
print('Accuracy of Decision tree on testing', dt2.score(X_test_scaled, y_test))

# Linear Discriminant Analysis 
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
lda = LinearDiscriminantAnalysis()
lda.fit(X_train_scaled,y_train)
print('Accuracy of Knn on training', lda.score(X_train_scaled, y_train))
print('Accuracy of Knn on testing', lda.score(X_test_scaled, y_test))



StarcoderdataJupyterScriptsDedupFiltered