# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/Tyler9937/DS-Unit-2-Applied-Modeling/blob/master/(U2W3A1_4).ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="b49ByxJ0oz8c" colab_type="code" colab={}
#importing pandas
#importing os as to work with local files
import pandas as pd
import os

#defining empty list
frames = []

#loading in all datasets
#datasets are named in such a way to be loaded in like this
#this current path as 619 seperate csv files to be read in
for i in range(1,2):
    df = pd.read_csv(r"C:\Users\tyler\Desktop\all_data\z (" + str(i) + ").csv" )
    frames.append(df)

# + id="lo00ExNEoz8g" colab_type="code" colab={}
#merging all of the datasets together
#merged datasets depend on ones read into frames
df = pd.concat(frames, sort=True)

# + id="1IL8sfAvoz8j" colab_type="code" colab={} outputId="80f286c7-01d4-4f04-e2c8-63eec923cfec"
# testing for duplicate rows in the dataframe
# Select duplicate rows except first occurrence based on all columns
duplicateRowsDF = df[df.duplicated(subset=None, keep='first')]

#prints out duplacted rows if any
print("Duplicate Rows except first occurrence based on all columns are :")
print(duplicateRowsDF)

# + id="1eiNjHkloz8n" colab_type="code" colab={} outputId="084e7ac9-940e-4292-ce34-369fc8765f21"
#checking df shape
df.shape

# + id="MHPy0OJvoz8q" colab_type="code" colab={} outputId="5269dcbd-3b8f-47a1-907f-763645f895bb"
df.dtypes

# + id="JbfIW_K8oz8s" colab_type="code" colab={} outputId="f62b44df-b9c8-472f-d7c5-c78638eaa30f"
#looking at df
pd.set_option('display.max_columns', None)
df.head()

# + id="ye1Kzljmoz8v" colab_type="code" colab={} outputId="540fbc5e-499f-4f6c-c387-4a8f3462bd2c"
#print(df['urls'][0])
print(df['profile'][0])
print(df['location'][2])
#print(df['creator'][0])

# + id="A_zFiYSDoz8x" colab_type="code" colab={} outputId="63bd7365-a13c-4524-972a-5c8de977c7f8"
df.isnull().sum()

# + id="Hbg8chbooz80" colab_type="code" colab={}
df.dropna(subset=['location'], inplace=True)

# + id="C2YtdQtyoz82" colab_type="code" colab={}


# + id="nGyz3-hIoz84" colab_type="code" colab={}
#creating a function that converts strings to dictionaries
import json 

def cat_to_dic(x):
   return json.loads(x) 

import ast
def loc_to_dic(x):
    return ast.literal_eval(str(x))



df['category_dic'] = df['category'].apply(cat_to_dic)
df['location_dic'] = df['location'].apply(cat_to_dic)

# + id="1XAl-i6Voz87" colab_type="code" colab={} outputId="0642c987-89ea-49c0-d625-949f0df5a3a9"
df.head()

# + id="KTfG1q8Hoz8-" colab_type="code" colab={}


# + id="rFY3YKADoz9A" colab_type="code" colab={}
def cat_to_id(x):
    return x['id']
def cat_to_name(x):
    return x['name']
def cat_to_slug(x):
    return x['slug']
def cat_to_short_names(x):
    return x['short_names']
def cat_to_displayable_name(x):
    return x['displayable_name']
def cat_to_localized_name(x):
    return x['localized_name']

# + id="UrWv5Ykpoz9C" colab_type="code" colab={}


# + id="vRKM8eX0oz9E" colab_type="code" colab={}
def cat_to_id(x):
    return x['id']
def cat_to_name(x):
    return x['name']
def cat_to_slug(x):
    return x['slug']
def cat_to_position(x):
    return x['position']
def cat_to_parent_id(x):
    return x['parent_id']
def cat_to_state(x):
    return x['state']
def cat_to_type(x):
    return x['type']
def cat_to_color(x):
    return x['color']


# + id="giWlJKJaoz9H" colab_type="code" colab={}
#Engineering new features
df['blurb_len'] = len(df['blurb'])
df['category_id'] = df['category_dic'].apply(cat_to_id)
df['category_name'] = df['category_dic'].apply(cat_to_name)
df['category_slug'] = df['category_dic'].apply(cat_to_slug)
df['category_position'] = df['category_dic'].apply(cat_to_position)
#df['category_parent_id'] = df['category_dic'].apply(cat_to_parent_id)
df['category_color'] = df['category_dic'].apply(cat_to_color)
df['is_usa'] = df['country'].str.contains('US')
df['is_usd'] = df['currency'].str.contains('USD')
df['name_length'] = len(df['name'])
df['slug_length'] = len(df['slug'])


df['location_id'] = df['location_dic'].apply(cat_to_id)
df['location_name'] = df['location_dic'].apply(cat_to_name)
#df['location_slug'] = df['location_dic'].apply(cat_to_slug)
#df['location_short_names'] = df['location_dic'].apply(cat_to_short_names)
#df['location_displayable_name'] = df['location_dic'].apply(cat_to_displayable_name)
df['location_state'] = df['location_dic'].apply(cat_to_state)


# + id="St9LfqCwoz9J" colab_type="code" colab={}
df['location_state'].fillna('test', inplace=True)

# + id="WXOUuaduoz9M" colab_type="code" colab={} outputId="f1e13df5-c4aa-4ab8-e157-1510e6e6342f"
df['location_state'].isnull().sum()

# + id="b1bV6vFtoz9O" colab_type="code" colab={}
#roughly seperating df into train, val, and test
#train = df[0:2000]
#val = df[2000:3000]
#test = df[3000:778]

# + id="5YKm1H8Roz9S" colab_type="code" colab={}

from sklearn.model_selection import train_test_split

# Split train into train & test
train, test = train_test_split(df, train_size=0.80, test_size=0.20, 
                              stratify=df['state'], random_state=42)

# Split train into train & val
train, val = train_test_split(train, train_size=0.80, test_size=0.20, 
                              stratify=train['state'], random_state=42)



# + id="aqqHrxj1oz9V" colab_type="code" colab={}
#bulk imorting libraries to be used in pipeline
import category_encoders as ce
from sklearn.feature_selection import f_regression, SelectKBest
from sklearn.impute import SimpleImputer
from sklearn.linear_model import Ridge
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier

#setting target as state
target = 'state'

#setting features as rest of columns minus columns that leak information 
#about the target
features = df.columns.drop([target, 'backers_count', 'converted_pledged_amount', 'fx_rate',
                           'is_backing', 'is_starrable', 'is_starred', 'permissions', 'pledged', 'profile',
                            'source_url', 'spotlight', 'state_changed_at', 'static_usd_rate',
                            'urls', 'usd_pledged', 'friends', 'usd_type', 'unseen_activity_count',
                            'category_dic', 'creator', 'currency_trailing_code', 'current_currency',
                           'disable_communication', 'location_dic', 'unread_messages_count',])

#removed temporayily
#'unread_messages_count', 'unseen_activity_count',

#creating our vars to be used in pipline
x_train = train[features]
y_train = train[target]
x_val = val[features]
y_val = val[target]
x_test = test[features]

# + id="CHoipBWwoz9Y" colab_type="code" colab={} outputId="4e908754-f0cd-4b69-c55d-1e3fc50ad1c6"
df['state'].isnull().sum()

# + id="ZyXjFsNJoz9a" colab_type="code" colab={} outputId="6f57dc30-3e52-459b-a718-fcbda42008d7"
#checking subset trains shape (features)
x_train.shape

# + id="hJjjlVHwoz9e" colab_type="code" colab={} outputId="20c73f93-5d41-4e3c-ba03-b61ca06a2e01"
#checking subset trains shape (target)
x_val.shape

# + id="fbDoFEIfoz9g" colab_type="code" colab={} outputId="373d5385-6ffb-4fed-8503-0e125f5be72b"
#finding the a baseline score for data

# there are five classes
# the code below shows the percentage
# of value_counts for each class

#using value_counts to find baseline
y_train.value_counts(normalize=True)

# + id="sE9snHESoz9j" colab_type="code" colab={} outputId="2842931a-bce8-4169-ee8c-91989d7ebb55"
#creating the pipline using Random forest classifer
#using ordinal encoder to make objtypes numeric
#using simple imputer to fill in nans
pipeline = make_pipeline(
    ce.OrdinalEncoder(), 
    SimpleImputer(strategy='median'), 
    RandomForestClassifier(n_estimators=100, n_jobs=1, random_state=0)
)

# Fit on train, score on val
pipeline.fit(x_train, y_train)

# + id="n8Io1z4Noz9l" colab_type="code" colab={} outputId="bc03cc1f-d669-4608-d64c-e91314637b55"
#printing out the validation accuracy
print('Train Accuracy', pipeline.score(x_train, y_train))
print('Validation Accuracy', pipeline.score(x_val, y_val))

# + id="gP5dG04noz9n" colab_type="code" colab={} outputId="b26bb32c-fa25-408e-adfc-f7fc8ed1c385"
#getting test accuracy
y_pred = pipeline.predict(x_test)
y_test = test[target]
print('Test Accuracy', pipeline.score(x_test, y_test))

# + id="_xxiK2nUoz9p" colab_type="code" colab={} outputId="5febf2bd-eaa0-4402-a4ed-768df7e81c8b"
#just analysising the data before plotting top features
print('X_train shape before encoding', x_train.shape)

encoder = pipeline.named_steps['ordinalencoder']
encoded = encoder.transform(x_train)

print('X_train shape after encoding', encoded.shape)

# + id="qlLpWKUuoz9q" colab_type="code" colab={} outputId="70d41344-95ed-47da-f29e-3de3ff68eb2d"
encoded.columns

# + id="GgPQnLqGoz9s" colab_type="code" colab={} outputId="53364bbc-66a1-42cc-f849-d1dce2b83fa8"
encoded.columns[0]

# + id="yeQsDiRMoz9v" colab_type="code" colab={} outputId="cf5e1ebb-d3ea-418f-a710-3020c89e5368"
#using matplotlib to plot best features
#Important top features are based of original model
#not the hyperparameter tuned model
# %matplotlib inline
import matplotlib.pyplot as plt

# Get feature importances
rf = pipeline.named_steps['randomforestclassifier']
importances = pd.Series(rf.feature_importances_, encoded.columns[0:36])

# Plot top n feature importances
n = 20
plt.figure(figsize=(10,n/2))
plt.title(f'Top {n} features')
importances.sort_values()[-n:].plot.barh(color='grey');

# + id="SrbcFxs6oz9x" colab_type="code" colab={}


# + id="MakF88Eeoz9y" colab_type="code" colab={}
#testing feature selection method from lecture

# + id="g8f3v9GYoz90" colab_type="code" colab={}
transformers = make_pipeline(
    ce.OrdinalEncoder(), 
    SimpleImputer(strategy='median')
)

# + id="OhR2rhS7oz92" colab_type="code" colab={} outputId="c6c75f9d-61a5-474d-8fe9-a541798fc8e6"
#testing out that feature selection tool learned today
x_train_transformed = transformers.fit_transform(x_train)
x_val_transformed = transformers.transform(x_val)

model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
model.fit(x_train_transformed, y_train)

# + id="CQ656rC_oz94" colab_type="code" colab={} outputId="3d92e085-3636-44cc-d761-1840d40d4b8f"
import eli5
from eli5.sklearn import PermutationImportance

# 1. Calculate permutation importances
permuter = PermutationImportance(
    model, 
    scoring='accuracy', 
    n_iter=5, 
    random_state=42
)

permuter.fit(x_val_transformed, y_val)

# + id="44rEswwNoz96" colab_type="code" colab={} outputId="4a61c2f4-f9de-492a-99dc-91e2dd9ebf12"
feature_names = x_val.columns.tolist()
pd.Series(permuter.feature_importances_, feature_names).sort_values()

# + id="RgyOSC6xoz98" colab_type="code" colab={} outputId="34b88c9b-ca6d-4271-d10a-b7900df08d1f"
# 2. Display permutation importances
eli5.show_weights(
    permuter, 
    top=None, # show permutation importances for all features
    feature_names=feature_names # must be a list
)

# + id="3wKg9_peoz9_" colab_type="code" colab={} outputId="591bad8a-e5f6-41a7-c9a8-754c647928af"
print('Shape before removing features:', x_train.shape, x_test.shape)

minimum_importance = 0
mask = permuter.feature_importances_ > minimum_importance
features = x_train.columns[mask]
x_train = x_train[features]

features = x_test.columns[mask]
x_test = x_test[features]

print('Shape after removing features:', x_train.shape, x_test.shape)

# + id="LYCw74X9oz-B" colab_type="code" colab={} outputId="7bbba8d0-deba-4754-e04f-867616f63f4d"
x_val = x_val[features]

pipeline = make_pipeline(
    ce.OrdinalEncoder(), 
    SimpleImputer(strategy='median'), 
    RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
)

# Fit on train, score on val
pipeline.fit(x_train, y_train)
print('Validation Accuracy', pipeline.score(x_val, y_val))

# + id="TtbXjIRToz-D" colab_type="code" colab={}


# + id="MvmoI9qKoz-F" colab_type="code" colab={}


# + id="tS6Pu__joz-H" colab_type="code" colab={} outputId="075bd55a-5634-4f59-dc12-04e94fa0fb9f"
from xgboost import XGBClassifier

pipeline2 = make_pipeline(
    ce.OrdinalEncoder(), 
    XGBClassifier(n_estimators=100, random_state=42, n_jobs=-1)
)

pipeline2.fit(x_train, y_train)

# + id="B2F4A9kSoz-J" colab_type="code" colab={} outputId="692b1560-5e50-4ec8-d1c1-26b5d2837a63"
from sklearn.metrics import accuracy_score
y_pred = pipeline2.predict(x_val)
print('Validation Accuracy', accuracy_score(y_val, y_pred))

# + id="iLS0o7oVoz-M" colab_type="code" colab={}


# + id="GSWqOfD-oz-R" colab_type="code" colab={}


# + id="cpWm_0pPoz-T" colab_type="code" colab={} outputId="3327bd43-9c43-4da6-a461-a6827bdfdc33"
# fit_transfom on train, transform on val
encoder = ce.OrdinalEncoder()
x_train_encoded = encoder.fit_transform(x_train)
x_val_encoded = encoder.transform(x_val)

model = XGBClassifier(
    n_estimators=1000,  # <= 1000 trees, depends on early stopping
    max_depth=7,        # try deeper trees because of high cardinality categoricals
    learning_rate=0.5,  # try higher learning rate
    n_jobs=-1
)

eval_set = [(x_train_encoded, y_train), 
            (x_val_encoded, y_val)]

model.fit(x_train_encoded, y_train, 
          eval_set=eval_set,
          eval_metric='merror', 
          early_stopping_rounds=50) # Stop if the score hasn't improved in 50 rounds

# + id="bB0RF6C5oz-V" colab_type="code" colab={} outputId="50508f6d-3b52-4af5-82fd-69d41a0f003f"
results = model.evals_result()
train_error = results['validation_0']['merror']
val_error = results['validation_1']['merror']
epoch = range(1, len(train_error)+1)
plt.plot(epoch, train_error, label='Train')
plt.plot(epoch, val_error, label='Validation')
plt.ylabel('Classification Error')
plt.xlabel('Model Complexity (n_estimators)')
#plt.ylim((0.18, 0.22)) # Zoom in
plt.legend();

# + id="BZG6YRe-oz-X" colab_type="code" colab={} outputId="1c5985b0-40db-4478-abd8-d494feb31ea1"
'''
So with about 6000(2files) rows the accuracy was about 62% to about 67%
with original train test val

with 6000(2 files) rows with train test val v2 we had as high as 72%
when 

50files accuracies at 79% on 74%when using xgboost

100files accuracies where at 87% and 74%when using xgboost

'''

# + id="vHexr1_xoz-Z" colab_type="code" colab={} outputId="bdafa280-9e35-4169-ce75-44bc170843b9"
'''
2files using the newly created location features we jumped to 73%-74%

50files had about 80% using xgboost had 74%. I think somthing is wrong with xgboost

100files had about 89% accuracy

300files 97.8% accuracy. data is powerful
'''

# + id="TInficGDoz-b" colab_type="code" colab={}


# + id="jpRqt7T3oz-d" colab_type="code" colab={} outputId="5b34ec65-7c70-4a30-b829-d3f299d1af8d"
df[features].isnull().sum()

# + id="au_sgLxyoz-f" colab_type="code" colab={} outputId="785baf97-f44a-4a82-ed4d-79177e81ab8c"
#creating a partial dependance plot for df

import category_encoders as ce
import seaborn as sns
from sklearn.ensemble import RandomForestClassifier



X = df[features]
y = df[target]

encoder = ce.OrdinalEncoder()
X_encoded = encoder.fit_transform(X)

model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
model.fit(X_encoded, y)



# + id="Ef4QAv3poz-j" colab_type="code" colab={} outputId="95119991-a964-4b45-a15a-754a590e16ca"
# Use Pdpbox
# %matplotlib inline
import matplotlib.pyplot as plt
from pdpbox import pdp
feature = 'deadline'
pdp_dist = pdp.pdp_isolate(model=model, dataset=X_encoded, model_features=features, feature=feature)
pdp.pdp_plot(pdp_dist, feature);

# + id="0xCwbkmMoz-l" colab_type="code" colab={}


# + id="mg0osOIpoz-n" colab_type="code" colab={}


# + id="_7mfJD54oz-o" colab_type="code" colab={}


# + id="5MRzymkzoz-q" colab_type="code" colab={} outputId="48f59fa1-3fe2-474a-f29f-e4f0319080b8"
x_train.shape

# + id="sdiyW8ehoz-r" colab_type="code" colab={}
import category_encoders as ce
from sklearn.impute import SimpleImputer
from sklearn.pipeline import make_pipeline
from xgboost import XGBClassifier

processor = make_pipeline(
    ce.OrdinalEncoder(), 
    SimpleImputer(strategy='median')
)

X_train_processed = processor.fit_transform(x_train)
X_val_processed = processor.transform(x_val)

eval_set = [(X_train_processed, y_train), 
            (X_val_processed, y_val)]

model = XGBClassifier(n_estimators=1000, n_jobs=-1)
model.fit(X_train_processed, y_train, eval_set=eval_set, eval_metric='auc', 
          early_stopping_rounds=10)

# + id="l1PDqnAxoz-t" colab_type="code" colab={} outputId="84db3a2b-507a-4e0a-b750-1d62ed6a2b33"
test_test = df[features]
row = x_test.iloc[[500]]
print(row)

# + id="Yv1NL_VCoz-w" colab_type="code" colab={} outputId="c7f4fbf1-e816-4d4d-ec63-8c69bba4a354"
x_test.shape

# + id="EHFq3xdooz-y" colab_type="code" colab={} outputId="495b1ce9-d001-4605-8505-fa9fa052f0b3"
x_train.shape

# + id="J6DNBy2goz-0" colab_type="code" colab={} outputId="6fa9acac-0b64-41ee-d41b-92f31fa0cf3e"
# STUDY/PRACTICE THIS CELL FOR THE SPRINT CHALLENGE
import shap

explainer = shap.TreeExplainer(model)
row_processed = processor.transform(row)
shap_values = explainer.shap_values(row_processed)

shap.initjs()
shap.force_plot(
    base_value=explainer.expected_value, 
    shap_values=shap_values, 
    features=row, 
    link='logit' # For classification, this shows predicted probabilities
)

# + id="ajq-ZUgcoz-2" colab_type="code" colab={}


# + id="4dVLCwlBoz-3" colab_type="code" colab={}


# + id="FpJ4onb1oz-5" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered