# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] papermill={"duration": 0.023942, "end_time": "2021-08-28T05:42:31.726133", "exception": false, "start_time": "2021-08-28T05:42:31.702191", "status": "completed"} tags=[]
# # Step 1: Import helpful libraries

# + papermill={"duration": 1.444048, "end_time": "2021-08-28T05:42:33.193152", "exception": false, "start_time": "2021-08-28T05:42:31.749104", "status": "completed"} tags=[]
# Familiar imports
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from math import ceil # round numbers up
# %matplotlib inline
# For ordinal encoding categorical variables, splitting data
from sklearn.preprocessing import OrdinalEncoder
from sklearn.model_selection import train_test_split

# For training random forest model
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold

import warnings
warnings.simplefilter(action='ignore', category=Warning)

# + [markdown] papermill={"duration": 0.022316, "end_time": "2021-08-28T05:42:33.238724", "exception": false, "start_time": "2021-08-28T05:42:33.216408", "status": "completed"} tags=[]
# # Step 2: Load the data
#
# Next, we'll load the training and test data.  

# + papermill={"duration": 5.067449, "end_time": "2021-08-28T05:42:38.329222", "exception": false, "start_time": "2021-08-28T05:42:33.261773", "status": "completed"} tags=[]
# Load the training data
train = pd.read_csv("../input/30-days-of-ml/train.csv", index_col=0)
test = pd.read_csv("../input/30-days-of-ml/test.csv", index_col=0)

# Preview the data
train.head()

# + [markdown] papermill={"duration": 0.023153, "end_time": "2021-08-28T05:42:38.378531", "exception": false, "start_time": "2021-08-28T05:42:38.355378", "status": "completed"} tags=[]
# ### EDA

# + [markdown] papermill={"duration": 0.023098, "end_time": "2021-08-28T05:42:38.425044", "exception": false, "start_time": "2021-08-28T05:42:38.401946", "status": "completed"} tags=[]
# let's see the distribution of the target variable

# + papermill={"duration": 0.828929, "end_time": "2021-08-28T05:42:39.278276", "exception": false, "start_time": "2021-08-28T05:42:38.449347", "status": "completed"} tags=[]
bins = np.arange(0, 12, 0.1)
sns.displot(train.target, height = 5, aspect = 2, bins = bins);

# + [markdown] papermill={"duration": 0.023741, "end_time": "2021-08-28T05:42:39.327413", "exception": false, "start_time": "2021-08-28T05:42:39.303672", "status": "completed"} tags=[]
# let's see the correlation 

# + papermill={"duration": 1.552497, "end_time": "2021-08-28T05:42:40.903969", "exception": false, "start_time": "2021-08-28T05:42:39.351472", "status": "completed"} tags=[]
plt.figure(figsize= (20, 15))
# Mask to hide upper-right part of plot as it is a duplicate
mask = np.transpose(np.tril(np.ones(train.corr().shape)))
sns.heatmap(train.corr(), annot = True, center = 0, cmap = 'RdBu', mask = mask);

# + [markdown] papermill={"duration": 0.0272, "end_time": "2021-08-28T05:42:40.959621", "exception": false, "start_time": "2021-08-28T05:42:40.932421", "status": "completed"} tags=[]
# We can see that target is weakly correlated with all features

# + papermill={"duration": 0.037262, "end_time": "2021-08-28T05:42:41.024797", "exception": false, "start_time": "2021-08-28T05:42:40.987535", "status": "completed"} tags=[]
num_cols = [col for col in train.columns if 'cont' in col] 
num_cols


# + papermill={"duration": 0.041009, "end_time": "2021-08-28T05:42:41.093608", "exception": false, "start_time": "2021-08-28T05:42:41.052599", "status": "completed"} tags=[]
def plot(data, cols, features_type, nrows, ncols, bins='auto', target=None, figsize=None,
         hspace=None, wspace=None, color = None):
    '''plot all features vs target or the distribution of features'''
    if figsize != None:
        plt.figure(figsize = figsize)
    for col, plot_num in zip(cols, list(range(1, len(cols)))):
        plt.subplot(nrows, ncols, plot_num)
        if hspace != None or wspace != None:
            plt.subplots_adjust(hspace = hspace, wspace = wspace)
            
        if features_type == 'numerical':
            if target != None:
                plt.scatter(data[col], data[target])
                plt.title(col)
            else:
                sns.histplot(data[col], bins=bins)
                
        if features_type == 'categorical':
            if target != None:
                sns.violinplot(data=data, y=col, x=target, color=color, inner='quartile');
            else:
                countplot_ratio(x = col, data = data, color = color)


# + [markdown] papermill={"duration": 0.026573, "end_time": "2021-08-28T05:42:41.147490", "exception": false, "start_time": "2021-08-28T05:42:41.120917", "status": "completed"} tags=[]
# Distributions of numerical features

# + papermill={"duration": 6.745767, "end_time": "2021-08-28T05:42:47.920242", "exception": false, "start_time": "2021-08-28T05:42:41.174475", "status": "completed"} tags=[]
n_cols = 4
n_rows = ceil(len(num_cols)/n_cols)
bins = np.arange(0, 1.3, 0.02)
plot(data=train, cols=num_cols, features_type='numerical', nrows=n_rows, ncols=n_cols, hspace=0.3, wspace=0.5, bins=bins,
    figsize = (15, 15))

# + [markdown] papermill={"duration": 0.02921, "end_time": "2021-08-28T05:42:47.978837", "exception": false, "start_time": "2021-08-28T05:42:47.949627", "status": "completed"} tags=[]
# let's see the distribution of continuous variable vs target

# + papermill={"duration": 15.232687, "end_time": "2021-08-28T05:43:03.241184", "exception": false, "start_time": "2021-08-28T05:42:48.008497", "status": "completed"} tags=[]
n_cols = 4
n_rows = ceil(len(num_cols)/n_cols)
plot(data=train, target='target', cols=num_cols, features_type='numerical', nrows=n_rows, ncols=n_cols, hspace=0.3,
    figsize = (15, 15))

# + papermill={"duration": 0.043143, "end_time": "2021-08-28T05:43:03.317279", "exception": false, "start_time": "2021-08-28T05:43:03.274136", "status": "completed"} tags=[]
# List of categorical columns
object_cols = [col for col in train.columns if 'cat' in col]
object_cols


# + [markdown] papermill={"duration": 0.032542, "end_time": "2021-08-28T05:43:03.382548", "exception": false, "start_time": "2021-08-28T05:43:03.350006", "status": "completed"} tags=[]
# Distributions of categorical features

# + papermill={"duration": 0.044187, "end_time": "2021-08-28T05:43:03.459555", "exception": false, "start_time": "2021-08-28T05:43:03.415368", "status": "completed"} tags=[]
# function to plot the distribution of categorical variable 
# since the countplot function show the counts of observations in each categorical bin using bars.
def countplot_ratio(x = None, data = None, hue = None, ax = None, color = None):
    # plot the variable
    ax = sns.countplot(x, data = data, hue = hue, ax = ax, color = color)
    # names of x labels
    ax.set_xticklabels(ax.get_xticklabels())
    # plot title
    ax.set_title(x + " Distribution")
    # total number of data which used to get the proportion
    total = float(len(data))
    # for loop to iterate on the patches
    for patch in ax.patches:
        # get the height of the patch which represents the number of observations.
        height = patch.get_height()
        # Put text on each patch with the proportion of the observations
        ax.text(patch.get_x()+patch.get_width()/2,height+4,'{:.2f}%'.format((height/total)*100),weight = 'bold',
                fontsize = 12,ha = 'center')


# + papermill={"duration": 5.087795, "end_time": "2021-08-28T05:43:08.580713", "exception": false, "start_time": "2021-08-28T05:43:03.492918", "status": "completed"} tags=[]
n_cols = 2
n_rows = ceil(len(object_cols)/n_cols)
base_color = sns.color_palette(n_colors=2)[1]
plot(data=train, cols=object_cols, features_type='categorical', nrows=n_rows, ncols=n_cols,
     hspace=0.5, figsize = (15, 20), color=base_color)

# + [markdown] papermill={"duration": 0.035522, "end_time": "2021-08-28T05:43:08.652751", "exception": false, "start_time": "2021-08-28T05:43:08.617229", "status": "completed"} tags=[]
# let's see the distribution of categorical variable vs target

# + papermill={"duration": 11.001525, "end_time": "2021-08-28T05:43:19.690084", "exception": false, "start_time": "2021-08-28T05:43:08.688559", "status": "completed"} tags=[]
n_cols = 3
n_rows = ceil(len(object_cols)/n_cols)
plot(data=train, target='target', cols=object_cols, features_type='categorical',
     nrows=n_rows, ncols=n_cols, hspace=0.5, figsize = (15, 20), color=base_color)

# + [markdown] papermill={"duration": 0.038533, "end_time": "2021-08-28T05:43:19.767555", "exception": false, "start_time": "2021-08-28T05:43:19.729022", "status": "completed"} tags=[]
# The next code cell separates the target (which we assign to `y`) from the training features (which we assign to `features`).

# + papermill={"duration": 0.100515, "end_time": "2021-08-28T05:43:19.906948", "exception": false, "start_time": "2021-08-28T05:43:19.806433", "status": "completed"} tags=[]
# Separate target from features
y = train['target']
features = train.drop(['target'], axis=1)

# Preview features
features.head()

# + [markdown] papermill={"duration": 0.039342, "end_time": "2021-08-28T05:43:19.985999", "exception": false, "start_time": "2021-08-28T05:43:19.946657", "status": "completed"} tags=[]
# # Step 3: Prepare the data
#
# Next, we'll need to handle the categorical columns (`cat0`, `cat1`, ... `cat9`).  

# + papermill={"duration": 4.206685, "end_time": "2021-08-28T05:43:24.232320", "exception": false, "start_time": "2021-08-28T05:43:20.025635", "status": "completed"} tags=[]
# ordinal-encode categorical columns
X = features.copy()
X_test = test.copy()
ordinal_encoder = OrdinalEncoder()
X[object_cols] = ordinal_encoder.fit_transform(features[object_cols])
X_test[object_cols] = ordinal_encoder.transform(test[object_cols])

# Preview the ordinal-encoded features
X.head()

# + [markdown] papermill={"duration": 0.039817, "end_time": "2021-08-28T05:43:24.313226", "exception": false, "start_time": "2021-08-28T05:43:24.273409", "status": "completed"} tags=[]
# Next, we break off a validation set from the training data.

# + [markdown] papermill={"duration": 0.039809, "end_time": "2021-08-28T05:43:24.393252", "exception": false, "start_time": "2021-08-28T05:43:24.353443", "status": "completed"} tags=[]
# # Step 4: Train a model
#
# Now that the data is prepared, the next step is to train a model.  

# + papermill={"duration": 2138.1021, "end_time": "2021-08-28T06:19:02.535300", "exception": false, "start_time": "2021-08-28T05:43:24.433200", "status": "completed"} tags=[]
fold_3_params = {'n_estimators':2500, 'max_depth' : 3,'learning_rate': 0.1,
              'colsample_bytree':0.13,
              'subsample':1, 'random_state':1, 'reg_alpha':25.9987, 'booster':'gbtree',
              'min_child_weight':1.1}

fold_6_params = {'n_estimators':2500, 'max_depth' : 3,'learning_rate': 0.13,
              'colsample_bytree':0.11,
              'subsample':0.99, 'random_state':1, 'reg_alpha':25.9987, 'booster':'gbtree',
              'min_child_weight':1.1}

fold_7_params = {'n_estimators':2500, 'max_depth' : 3,'learning_rate': 0.1,
              'colsample_bytree':0.13,
              'subsample':0.99, 'random_state':1, 'reg_alpha':25.9987, 'booster':'gbtree',
              'min_child_weight':1.1}

fold_8_params = {'n_estimators':2500, 'max_depth' : 3,'learning_rate': 0.1,
              'colsample_bytree':0.13,
              'subsample':0.99, 'random_state':1, 'reg_alpha':25.9987, 'booster':'gbtree',
              'min_child_weight':1.1}
# fold parameters
n_splits = 10
kfold  = KFold(n_splits=n_splits, shuffle=True, random_state=42)

# To store 'out of fold' predictions, we create an array of zeros
oof_predictions = np.zeros((X.shape[0],))
predictions = 0
total_mean_RMSE = 0

# Generating folds and making training and prediction for each of 10 folds
for fold_num, (train_ix, valid_ix) in enumerate(kfold.split(X)):
    params = {'n_estimators':2500, 'max_depth' : 3,'learning_rate': 0.13,
              'colsample_bytree':0.13,
              'subsample':0.99, 'random_state':1, 'reg_alpha':25.9987, 'booster':'gbtree',
              'min_child_weight':1.1}
    
    X_train, X_valid = X.iloc[train_ix], X.iloc[valid_ix]
    y_train, y_valid = y.iloc[train_ix], y.iloc[valid_ix]
    if fold_num == 3:
        params = fold_3_params
    if fold_num == 6:
        params = fold_6_params
    if fold_num == 7:
        params = fold_7_params
    if fold_num == 8:
        params = fold_8_params
    model = XGBRegressor(**params)
    model.fit(X_train, y_train,
              verbose=False,
              # These parameters stop training before the model starts overfitting. 
              eval_set=[(X_train, y_train),(X_valid, y_valid)],
              eval_metric="rmse",
              early_stopping_rounds=100,
              )
    # Getting mean test data predictions 
    predictions += model.predict(X_test) / n_splits
    
    # Now, we get validation data preds. Each fold model get predictions from unseen data.
    # After all folds model get predictions, it will be filled with unseen data preds.
    # We will use it to evaluate hyperparameters performance only.
    oof_predictions[valid_ix] = model.predict(X_valid)
    
    # score for a fold model
    fold_RMSE = np.sqrt(mean_squared_error(y_valid, oof_predictions[valid_ix]))
    print(f"Fold {fold_num} RMSE: {fold_RMSE}")

    # Getting total mean of scores of all fold models 
    total_mean_RMSE += fold_RMSE / n_splits
    
print(f"\nOverall RMSE: {total_mean_RMSE}")

# + papermill={"duration": 0.054178, "end_time": "2021-08-28T06:19:02.637197", "exception": false, "start_time": "2021-08-28T06:19:02.583019", "status": "completed"} tags=[]
from sklearn.model_selection import GridSearchCV
def  Hyperparameter_tuning(params):
        '''We use this function to get the best hyperparameters
        Parameters
        ----------
        param: dict
        A dictionary of hyperparameters names and lists of possible values of it
        example:
        param = { 'max_depth': [3,6,10],
           'learning_rate': [0.01, 0.05, 0.1],
           'n_estimators': [100, 500, 1000],
           'colsample_bytree': [0.3, 0.7]}'''
        params = params
        model = XGBRegressor(tree_method = 'gpu_hist')
        clf = GridSearchCV(estimator=model, 
                           param_grid=params,
                           scoring='neg_mean_squared_error', 
                           verbose=2)
        clf.fit(X, y)
        print("Best parameters:", clf.best_params_)
        print("Lowest RMSE: ", (-clf.best_score_)**(1/2.0))


# + [markdown] papermill={"duration": 0.043018, "end_time": "2021-08-28T06:19:02.724285", "exception": false, "start_time": "2021-08-28T06:19:02.681267", "status": "completed"} tags=[]
# In the code cell above, we set `squared=False` to get the root mean squared error (RMSE) on the validation data.
#
# # Step 5: Submit to the competition
#
# We'll begin by using the trained model to generate predictions, which we'll save to a CSV file.

# + papermill={"duration": 0.672929, "end_time": "2021-08-28T06:19:03.441027", "exception": false, "start_time": "2021-08-28T06:19:02.768098", "status": "completed"} tags=[]
output = pd.DataFrame()
output["id"] = test.index
output["target"] = predictions
# Save the predictions to a CSV file
output.to_csv('submission.csv', index=False, header=output.columns)
output.head()

StarcoderdataJupyterScriptsDedupFiltered