# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/NikuDubenco/DS-Unit-2-Regression-2/blob/master/Nicolae_Dubenco_Cross_Validation%2C_Hyperparameter_Optimization.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + id="aHtkCrkosSh3" colab_type="code" outputId="a419ec5a-e342-4ba8-be61-b824dd804e7d" colab={"resources": {"http://localhost:8080/nbextensions/google.colab/files.js": {"data": "<KEY> "ok": true, "headers": [["content-type", "application/javascript"]], "status": 200, "status_text": ""}}, "base_uri": "https://localhost:8080/", "height": 366}
from google.colab import files
files.upload()

# + id="NMNTHMMjsnS5" colab_type="code" outputId="e749aa59-5b66-46dc-c4ce-aa11001987d8" colab={"base_uri": "https://localhost:8080/", "height": 85}
# !unzip caterpillar-tube-pricing.zip

# + id="rqV32FDltLAd" colab_type="code" outputId="bed55487-3100-4143-b2e2-e20db8027f65" colab={"base_uri": "https://localhost:8080/", "height": 408}
# !unzip data.zip

# + [markdown] id="zbwBxbGptr_q" colab_type="text"
# ### Wrangle data

# + id="tXe9Z7Zktucw" colab_type="code" colab={}
import category_encoders as ce
from glob import glob
import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestRegressor
from sklearn.pipeline import make_pipeline

# + id="vkV7KOfKuIp_" colab_type="code" colab={}
#read data
train = pd.read_csv('competition_data/train_set.csv')
test = pd.read_csv('competition_data/test_set.csv')
tube = pd.read_csv('competition_data/tube.csv')
materials = pd.read_csv('competition_data/bill_of_materials.csv')
components = pd.read_csv('competition_data/components.csv')
comp = pd.concat((pd.read_csv(path) for path in glob('competition_data/comp_*.csv')), sort=False)

# + id="w4IFVOP1vdol" colab_type="code" outputId="6c274e4c-47f6-49a2-c178-ad0d9f2d178f" colab={"base_uri": "https://localhost:8080/", "height": 317}
comp.describe()

# + id="KqDVZ03Mvf6i" colab_type="code" colab={}
# get a tidy list of the component types in each tube assembly
assembly_components = materials.melt(id_vars='tube_assembly_id',
                                    value_vars=[f'component_id_{n}' for n in range(1,9)])
assembly_components = assembly_components.sort_values(
    by='tube_assembly_id').dropna().rename(columns={'value': 'component_id'})

assembly_component_types = assembly_components.merge(components, how='left')

# + id="K9ZKrmeYwtVY" colab_type="code" colab={}
table = pd.crosstab(assembly_component_types['tube_assembly_id'],
                   assembly_component_types['component_type_id'])
table = table.reset_index()

# + id="sC0rIBZNxWNc" colab_type="code" outputId="eec88a5d-1bf2-4d97-fd4d-7be6d7ac2dd7" colab={"base_uri": "https://localhost:8080/", "height": 2006}
table

# + id="6ZttDsAjxeEC" colab_type="code" colab={}
features = ['component_id', 'component_type_id', 'orientation', 'unique_feature', 'weight']
comp = comp[features]
comp['orientation'] = (comp['orientation']=='Yes').astype(int)
comp['unique_feature'] = (comp['unique_feature']=='Yes').astype(int)
comp['weight'] = comp['weight'].fillna(comp['weight'].median())

# + id="xTVNaW62UjHI" colab_type="code" colab={}
# get agg features for all components in each tube assembly
# this code is a little complex, but we discussed in detail last lesson

materials['components_total'] = sum(materials[f'quantity_{n}'].fillna(0) for n in range(1,9))
materials['components_distinct'] = sum(materials[f'component_id_{n}'].notnull().astype(int) for n in range(1,9))
materials['orientation'] = 0
materials['unique_feature'] = 0
materials['weight'] = 0

for n in range(1,9):
  materials = materials.merge(comp, how='left',
                             left_on=f'component_id_{n}',
                             right_on='component_id',
                             suffixes=('', f'_{n}'))
  
for col in materials:
  if 'orientation' in col or 'unique_feature' in col or 'weight' in col:
    materials[col] = materials[col].fillna(0)
    
materials['orientation'] = sum(materials[f'orientation_{n}'] for n in range(1,9))
materials['unique_feature'] = sum(materials[f'unique_feature_{n}'] for n in range(1,9))
materials['weight'] = sum(materials[f'weight_{n}'] for n in range(1,9))

features = ['tube_assembly_id', 'orientation', 'unique_feature', 'weight', 'components_total',
           'components_distinct', 'component_id_1']
materials = materials[features]

# + id="SDfYkCNYYdCu" colab_type="code" colab={}
# extract year from quote date
train['quote_date_year'] = pd.to_datetime(train['quote_date'], infer_datetime_format=True).dt.year
test['quote_date_year'] = pd.to_datetime(test['quote_date'], infer_datetime_format=True).dt.year

# + id="tfCIKzgvZTPI" colab_type="code" colab={}
# merge data
train = (train
        .merge(tube, how='left')
        .merge(materials, how='left')
        .merge(table, how='left')
        .fillna(0))

test = (test
       .merge(tube, how='left')
       .merge(materials, how='left')
       .merge(table, how='left')
       .fillna(0))

# arrange X matrix and y vector.
# drop 'tube_assembly_id' because our goal is to predict unknown assemblies,
# and no tube assembly_id's are shared between the train and test sets
target = 'cost'
features = train.columns.drop([target, 'tube_assembly_id'])
X_train = train[features]
y_train = train[target]
X_test = test[features]

# + id="eAielJWMbKcm" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 170} outputId="3c518490-36a9-4fc2-94e1-c6933bd2d557"
y_train.describe()

# + id="wFCmONOgbNMl" colab_type="code" colab={}
# log-transform the target
y_train_log = np.log1p(y_train)

# make pipeline
pipeline = make_pipeline(ce.OrdinalEncoder(),
                        RandomForestRegressor(n_estimators=100, n_jobs=-1, random_state=42))

# + [markdown] id="jkkp9KQvb719" colab_type="text"
# ### cross_val_score

# + id="AL3TCjEhb43i" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 68} outputId="4fd174d7-60c6-4748-c8c9-d2647a35ea4d"
# %%time
from sklearn.model_selection import cross_val_score

k=3
groups = train['tube_assembly_id']
scores = cross_val_score(pipeline, X_train, y_train_log, cv=k,
                       scoring='neg_mean_squared_error', groups=groups)

print(f'RMSLE for {k} folds:', np.sqrt(-scores))

# + id="Ns5eG95ecx2q" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 153} outputId="b151e7c1-ee63-4de8-802f-ba374e50134b"
print('Model Hyperparameters:')
print(pipeline.named_steps['randomforestregressor'])

# + [markdown] id="L56h5dhpdRY5" colab_type="text"
# ### validation curve

# + id="mW_BXlJFdN2s" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 283} outputId="e7f47b77-e587-44ce-bb50-c3b040d71a78"
# %matplotlib inline
import matplotlib.pyplot as plt
from sklearn.model_selection import validation_curve
from sklearn.tree import DecisionTreeRegressor

pipeline = make_pipeline(ce.OrdinalEncoder(), DecisionTreeRegressor())

depth = range(1, 15, 2)

train_scores, val_scores = validation_curve(pipeline, X_train, y_train_log, 
                                           param_name='decisiontreeregressor__max_depth',
                                           param_range=depth, scoring='neg_mean_squared_error',
                                           cv=2, groups=groups)

train_rmsle = np.sqrt(-train_scores)
val_rmsle = np.sqrt(-val_scores)

plt.plot(depth, np.mean(train_rmsle, axis=1), color='blue', label='training error')
plt.plot(depth, np.mean(val_rmsle, axis=1), color='r', label='validation error')
plt.xlabel('depth')
plt.ylabel('rmsle')
plt.legend();

# + id="ASQgGm5FfdrI" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 513} outputId="6f888b6e-302b-4ca3-e85c-4648d56891a0"
# use scikit-learn for hyperparameter optimization
from scipy.stats import randint, uniform
from sklearn.model_selection import RandomizedSearchCV

pipeline = make_pipeline(ce.OrdinalEncoder(),
                        RandomForestRegressor(random_state=42))

param_distributions = {
    'randomforestregressor__n_estimators': randint(50,500),
    'randomforestregressor__max_features': uniform(),
    'randomforestregressor__min_samples_leaf': [1,10,100]
}

search = RandomizedSearchCV(pipeline,
                           param_distributions=param_distributions,
                           n_iter=5,
                           cv=2,
                           scoring='neg_mean_squared_error',
                           verbose=10,
                           return_train_score=True,
                           n_jobs=-1)

search.fit(X_train, y_train_log, groups=groups)

# + id="cgtXCTIHiSG9" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 71} outputId="98d9dea3-f561-4176-fb69-33825699851a"
print('Best hyperparameters', search.best_params_)
print('Cross-validation RMSLE', np.sqrt(-search.best_score_))

# + [markdown] id="abC7LKznjGWv" colab_type="text"
# ### xgboost

# + id="Zd21l5wTjDr3" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 428} outputId="7bfd681a-eba6-4b72-9922-1142fd348718"
from xgboost import XGBRegressor
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

pipeline = make_pipeline(ce.OrdinalEncoder(),
                        XGBRegressor(random_state=42))

param_distributions = {
    'xgbregressor__n_estimators': randint(500, 1000),
    'xgbregressor__max_depth': randint(3, 7)
}

search = RandomizedSearchCV(pipeline,
                           param_distributions=param_distributions,
                           n_iter=5,
                           cv=2,
                           scoring='neg_mean_squared_error',
                           return_train_score=True,
                           n_jobs=-1)

search.fit(X_train, y_train_log, groups=groups)

# + id="3cS-hGVMksQF" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 51} outputId="6e1c6375-7380-40f1-8a77-fd4ff98c1ec5"
print('Best hyperparameters', search.best_params_)
print('Cross-validation RMSLE', np.sqrt(-search.best_score_))

# + id="huxePHU_lKE3" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 309} outputId="d0f93a37-6539-4539-c6c3-1908ab110279"
results_df = pd.DataFrame(search.cv_results_)
results_df.head()

# + id="912W4jAulVB8" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 309} outputId="8dc8e7be-88c6-40f0-801f-5d8e70dacc78"
results_df.sort_values(by='rank_test_score')

# + [markdown] id="X5_GVIidliI6" colab_type="text"
# ### make prediction

# + id="MFHfl1_DldU6" colab_type="code" colab={}
pipeline = search.best_estimator_
y_pred_log = pipeline.predict(X_test)
y_pred = np.expm1(y_pred_log) # convert from log-dollars to dollars
submission = pd.read_csv('sample_submission.csv')
submission['cost'] = y_pred
submission.to_csv('submission.csv', index=False)

# + id="oj5vOFZVmTgF" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered