# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ##### ML jupyter notebook
# ##### This notebook is dedicated to my finding trends and analysis of accidents

# #### 1. LIBRARIES

# !pip install seaborn
# !pip install sklearn

# +
# importing the necessary libraries
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# %matplotlib inline

pd.set_option('display.max_rows', 200)
pd.set_option('display.max_columns', 200)
plt.style.use('ggplot')

# Import LogisticRegression
from sklearn.linear_model import LogisticRegression

# used for both linear and multiple regression
from sklearn.linear_model import LinearRegression 

from sklearn.preprocessing import PolynomialFeatures

# Import RandomForestClassifier
from sklearn.ensemble import RandomForestClassifier

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
# -

# see if we have the right file in working directory
for files in os.listdir():
    print(files)

# # Need to drop some columns cause of recurring memory issues
# #### "Sunrise_Sunset" shows the period of day (i.e day or night) based on sunrise/sunset.
# #### "Visibility(mi)" is being measured in miles

# +
# useful columns
# USING THESE FEATURES GAVE 94.7% AND 93.4% accuracy for logistic regression and RANDOM FOREST
# useful_colmns = ["Source", "Severity", "Start_Time", "End_Time", "City", "County", "State", 
#               "Visibility(mi)", "Weather_Condition", "Sunrise_Sunset"]  

useful_colmns = ['Source','TMC','Severity','Start_Time','End_Time','Start_Lng',
                 'Start_Lat','Distance(mi)','Side','City','County','State','Temperature(F)',
                 'Humidity(%)','Pressure(in)', 'Visibility(mi)', 'Wind_Direction',
                 'Weather_Condition','Amenity','Bump','Crossing','Give_Way','Junction',
                 'No_Exit','Railway','Roundabout','Station','Stop','Traffic_Calming',
                 'Traffic_Signal','Turning_Loop','Sunrise_Sunset']

df = pd.read_csv("US_Accidents_June20.csv", usecols = useful_colmns, nrows = 500000)

# +
# df.dtypes
# -

# df.rename(columns={'Visibility(mi)': 'Visibility', 'Distance(mi)': 'Distance', ''}, inplace = True)
df.tail(10)

# #### The code below works for reading in chunks but I keep running into memory issues and TextToFileReader problems

# +
# The df contains up to 3.6 million of data 
# chunkSIZE = 500000 # we're reading our data in chunks because of how large our data set is.
# dataList = []     ## list to hold the chunks for concatenation.

# # for chunk in pd.read_csv("US_Accidents_June20.csv", chunksize = chunkSIZE):
# #     dataList.append(chunk)

# # df = pd.concat(dataList, axis=0)
# chunk1 = pd.read_csv("US_Accidents_June20.csv", chunksize = 1000)
# -

# #### We need to do some filtering and preprocessing to our data to get rid of redundant entries

df.shape

# Steps in dealing with missing data values
missing_data = df.isnull()
missing_data.head(10) ## checking the tail to see if any data is missing

# for each column in the df, we wanna look through to find any missing data
for column in missing_data.columns.values.tolist():
    print(column)
    print(missing_data[column].value_counts())
    print("")

# Based on the output above, ONLY 4 columns contain missing data including
# City, Visibility(mi), Weather_Condition, Sunrise_Sunset
#
# ## Deal with missing data
#
# ##### How to deal with missing data?
#  ###### 1. drop data
#    a) drop the whole row
#    b) drop the whole column
#  ###### 2. replace data
#    a) replace it by mean
#    b) replace it by frequency
#    c) replace it based on other functions e.g median

# +
# Replace by frequency for City column
# df['City'].value_counts()

# +
# df['City'].value_counts().idxmax()
# -

# #### Since "Houston" has the most occurence for the City column, we replace 112 missing data with "Houston"
# making the new total for "Houston" 101352

# +
# replace the missing 'City' values by the most frequent 
# df["City"].replace(np.nan, "Houston", inplace=True)

# +
# df['City'].value_counts()
# -

# <h4>Calculate the mean value for the  'Visibility' column:</h4>

avg_Visibility = df['Visibility(mi)'].astype('float').mean(axis=0)
print("Average Visibility:", avg_Visibility)

# <h4>Replace "NaN" by mean value:</h4>

df['Visibility(mi)'].replace(np.nan, avg_Visibility, inplace=True)

# ##### Checking if preprocessing has updated in our original dataframe

missing_data2 = df.isnull()
for column in missing_data2.columns.values.tolist():
    print(column)
    print(missing_data2[column].value_counts())
    print("")

# Replace by frequency for Weather_Condition column
df['Weather_Condition'].value_counts()

# #### Since "Clear" has the most occurence for the Weather_Condition column, we replace 76138 missing data with "Clear"
# making the new total for "Clear" 884340

df['Weather_Condition'].value_counts().idxmax()
# df.head(5)

# replace the missing 'City' values by the most frequent 
df["Weather_Condition"].replace(np.nan, "Clear", inplace=True)

df['Weather_Condition'].value_counts()

# replace Sunrise/Sunset column with the most occuring between "Night" and "Day"
df['Sunrise_Sunset'].value_counts().idxmax()

# replace the missing 'NAN' values by the Day 
df["Sunrise_Sunset"].replace(np.nan, "Day", inplace=True)

missing_data3 = df.isnull()
for column in missing_data3.columns.values.tolist():
    print(column)
    print(missing_data3[column].value_counts())
    print("")

# ##### Apply the concept of frequence and averaging to the outliers in the columns wind_direction, pressure, humidity and temperature respectively

# +
freq_diretion = df['Wind_Direction'].value_counts().idxmax()
df["Wind_Direction"].replace(np.nan, freq_diretion, inplace=True)

avg_Pressure = df['Pressure(in)'].astype('float').mean(axis=0)
df['Pressure(in)'].replace(np.nan, avg_Pressure, inplace=True)

avg_Humidity = df['Humidity(%)'].astype('float').mean(axis=0)
df['Humidity(%)'].replace(np.nan, avg_Humidity, inplace=True)

avg_Temperature = df['Temperature(F)'].astype('float').mean(axis=0)
df['Temperature(F)'].replace(np.nan, avg_Temperature, inplace=True)
# -

missing_data4 = df.isnull()
for column in missing_data4.columns.values.tolist():
    print(column)
    print(missing_data4[column].value_counts())
    print("")

# DROP THE REMAINING EMPTY CELLS FROM THE COLUMN CITY TO CONCLUDE THE DATA CLEANSING SECTION
df.isnull().mean()

# So in total we should only be dropping 22 rows from the city column
df.dropna(subset=df.columns[df.isnull().mean()!=0], how='any', axis=0, inplace=True)
df.shape

# ## At this point in the script, all the NAN values have wither been replaced by most occuring or average values of the column in question

df.head(5)
df.shape

# +
# df.to_csv('Output.csv', index=True)
# -

# #### Let's extract the year, month, day, weekday and time for easy processing of this values

df.head(5)

# +
# Convert Start_Time and End_Time to datetypes
df['Start_Time'] = pd.to_datetime(df['Start_Time'], errors='coerce')
df['End_Time'] = pd.to_datetime(df['End_Time'], errors='coerce')

# Extract year, month, day, hour and weekday
df['Year']=df['Start_Time'].dt.year
df['Month']=df['Start_Time'].dt.strftime('%b')
df['Day']=df['Start_Time'].dt.day
df['Hour']=df['Start_Time'].dt.hour
df['Weekday']=df['Start_Time'].dt.strftime('%a')

# Extract the amount of time in the unit of minutes for each accident, round to the nearest integer
td='Time_Duration(min)'
df[td]=round((df['End_Time']-df['Start_Time'])/np.timedelta64(1,'m'))
df.info()
# -

# ##### Validate to make sure time duration is not negative for any given row

df[td][df[td] <=0]

# +
# Since we have some entries with negative time values, let's drop those rows by changing their 
# entries to NAN values and then using dropna to drop the values respectively
negative_time = df[td] <= 0

# Set outliers to NAN
df[negative_time] = np.nan

# Drop rows with negative td
df.dropna(subset=[td], axis=0, inplace=True)
df.info()
# -

# Validate if there's still misssing values and drop them
df.head()
# df.isnull().mean()

df.info()

# Double Chack to make sure we have non-negative time durations
df[td][df[td]<=0]

# +
# Remove outliers for Time_Duration(min): n * standard_deviation (n=3), backfill with median
# n=3

# median = df[td].median()
# std = df[td].std()
# outliers = (df[td] - median).abs() > std*n

# # Set outliers to NAN
# df[outliers] = np.nan

# # Fill NAN with median
# df[td].fillna(median, inplace=True)

# df.info()

# +
# Since our data is now properlly formatted and ready for analysis, 

# Set the list of features to include in Machine Learning
# features=['Source','TMC','Severity','Start_Lng','Start_Lat','Distance(mi)',
#              'Side','City','County','State','Temperature(F)','Humidity(%)',
#              'Pressure(in)', 'Visibility(mi)', 'Wind_Direction','Weather_Condition',
#              'Amenity','Bump','Crossing','Give_Way','Junction','No_Exit','Railway',
#              'Roundabout','Station','Stop','Traffic_Calming','Traffic_Signal','Turning_Loop',
#              'Sunrise_Sunset','Hour','Weekday', 'Time_Duration(min)']

# max hour is 23 which means time is in the 24hour format
features=['Severity','Start_Lng','Start_Lat','Temperature(F)','Humidity(%)','Pressure(in)','Hour']

df_final = df[features].copy()
df_final

# +
# # Set state
# state='PA'

# # Select the state of Pennsylvania
# df_state=df_final.loc[df_final.State==state].copy()
# df_state.drop('State',axis=1, inplace=True)

# # Set county
# county='Montgomery'

# # Select the county of Montgomery from the state of PA
# df_county=df_state.loc[df_state.County==county].copy()
# df_county.drop('County',axis=1, inplace=True)
# df_county.shape

# +
# Generate dummies for categorical data
# df_county_dummy = pd.get_dummies(df_county, drop_first=True)
# df_county_dummy.head()

# +
df_analysis = df_final

# Target for the prediction
target='Severity'

# Create arrays for the features and the response variable

# set X and y
X = df_analysis.drop(target, axis=1)
y = df_analysis[[target]]

# Split the data set into training and testing data sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
# y.head()
# -

# ## Predict the accident severity with various supervised machine learning algorithms 

# +
# List of classification algorithms
algo_lst=['Logistic Regression', 'Linear Regression', 'Miltiple-Linear Regression', 'Polynomial Regression']

# Initialize an empty list for the accuracy for each algorithm
accuracy_lst=[]
X.head()
# -

# #### A. Logistic Regression

# +
# col_mask=df_analysis.isnull().any(axis=0) 
# col_mask

# row_mask=df.isnull().any(axis=1)
# row_mask

# np.isfinite(X_train).all()
X_train.isnull().values.all()

# +
# Logistic regression
logreg = LogisticRegression(random_state=0)
logreg.fit(X_train,y_train)
y_pred = logreg.predict(X_test)

# Get the accuracy score
acc=accuracy_score(y_test, y_pred)

# Append to the accuracy list
accuracy_lst.append(acc)

print("[Logistic regression algorithm] accuracy_score: {:.3f}.".format(acc))
# -

y_test

y_pred

# #### B. Linear Regression

# +
# # # set X and y
# y1 = df_county[['Severity']]
# X1 = df_county[['Time_Duration(min)']]

# # LinearRegression()
# lr = LinearRegression()
# lr.fit(X1, y1)

# acc1 = lr.score(X1, y1)
# print(acc1)
# yhat = lr.predict([[314]])
# print(yhat)

# accuracy_lst.append(acc1)
# # features = ['Source','Severity','City','County','State','Visibility', 
# #              'Weather_Condition','Hour','Weekday', 'Time_Duration(min)']
# -

# #### C. Multiple Linear Rgression

# +
# accuracy_lst

# +
# # set X and y
# y2 = df_analysis[['Severity']]
# X2 = df_analysis.drop(target, axis=1)

# lr2 = LinearRegression()
# lr2.fit(X2, y2)

# acc2 = lr2.score(X2, y2)
# print(acc2)
# yhat2 = lr2.predict(X2)
# print(yhat2)

# accuracy_lst.append(acc2)

# +
# accuracy_lst
# -

# #### 3. Polynomial Features on Multiple Linear Regression

# ###### This method doesn't work properly cause it raises memory problems

# +
# Rsqu_test = []

# order = [1,2,3,4,5]
# # lr = LinearRegression()

# # for n in order:
# #     pr = PolynomialFeatures(degree=n)
    
# #     X_train_pr = pr.fit_transform(X_train)
# #     X_test_pr = pr.fit_transform(X_test)
    
# #     lr.fit(X_train_pr, y_train)
    
# #     Rsqu_test.append(lr.score(X_test_pr, y_test))
   
# lr = LinearRegression()
# pr = PolynomialFeatures(degree=3)

# X_train_pr = pr.fit_transform(X_train)
# X_test_pr = pr.fit_transform(X_test)

# lr.fit(X_train_pr, y_train)

# Rsqu_test.append(lr.score(X_test_pr, y_test))
# -

# ##### 4. Random Forest

# +
# Random Forest algorithm
rnd_Clf = RandomForestClassifier(n_estimators=100)

#Train the model using the training sets 
rnd_Clf.fit(X_train,y_train)

# And predict using the testing set y_pred=rnd_Clf.predict(X_test)
y_pred = rnd_Clf.predict(X_test)


# Get the accuracy score
acc = accuracy_score(y_test, y_pred)

# Append to the accuracy list
accuracy_lst.append(acc)


# Model Accuracy, how often is the classifier correct?
print("[Randon forest algorithm] accuracy_score: {:.3f}.".format(acc))
# -

accuracy_lst

# +
# Let's observe the features that really contribute to the accuracy of our prediction
important_features = pd.Series(rnd_Clf.feature_importances_,index=X.columns).sort_values(ascending=False)

# Creating a bar plot, displaying only the top k features
k=11
sns.barplot(x=important_features[:k], y=important_features.index[:k])
# Add labels to your graph
plt.xlabel('Feature Importance Score')
plt.ylabel('Features')
plt.title("Visualizing Important Features")
plt.legend()
plt.show()

# +
# NEED TO PICKLE OUR MODEL
# import pickle 
# example_dict = {1:"6", 2:"3", 3:"F"}

# pickle_out = open("dict.picle", "wb")
# pickle.dump(example_dict, pickle_out)
# pickle_out.close()


# # read back pickled object into the program again
# pickle_in = open("dic.pickle", "rb")
# loaded_dict = picle.load(pickle_in)

new_data = [[43.2557, -79.8711, 6, 10.04, 50, 65]]
X_new = pd.DataFrame(new_data, columns = ["Start_Lat", "Start_Lng", "Hour", "Pressure(in)", "Temperature(F)", "Humidity(%)"]) 

# And predict using the testing set y_pred=rnd_Clf.predict(X_test)
y_new = rnd_Clf.predict(X_new)
print(y_new)
# -





StarcoderdataJupyterScriptsDedupFiltered