# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] papermill={"duration": 0.017145, "end_time": "2021-03-11T21:01:41.723736", "exception": false, "start_time": "2021-03-11T21:01:41.706591", "status": "completed"} tags=[]
# # Import dependencies

# + papermill={"duration": 6.576545, "end_time": "2021-03-11T21:01:48.316397", "exception": false, "start_time": "2021-03-11T21:01:41.739852", "status": "completed"} tags=[]
import numpy as np
import pandas as pd
import pandas_profiling
import time

from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score
from sklearn.preprocessing import LabelEncoder, OrdinalEncoder
from sklearn.compose import ColumnTransformer
from sklearn.metrics import roc_auc_score, average_precision_score, log_loss

from hyperopt import fmin, hp, tpe, Trials, space_eval
from hyperopt.pyll import scope as ho_scope
from hyperopt.pyll.stochastic import sample as ho_sample
from functools import partial

from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from catboost import CatBoostClassifier

from imblearn.pipeline import Pipeline
#from imblearn.over_sampling import ADASYN
from imblearn.under_sampling import OneSidedSelection, NeighbourhoodCleaningRule, TomekLinks, RandomUnderSampler

import category_encoders as ce


# + papermill={"duration": 0.025132, "end_time": "2021-03-11T21:01:48.359859", "exception": false, "start_time": "2021-03-11T21:01:48.334727", "status": "completed"} tags=[]
def evalue_model(model, y_test, X_test, model_name):
    
    yhat_prob = [x[1] for x in model.predict_proba(X_test)]
    
    results = {'model': model_name,
               'auc': roc_auc_score(y_true = y_test, y_score = yhat_prob),
               'aucpr': average_precision_score(y_true = y_test, y_score = yhat_prob),
               'logloss': log_loss(y_test, yhat_prob)}
    
    return results


# + papermill={"duration": 3.599286, "end_time": "2021-03-11T21:01:51.976202", "exception": false, "start_time": "2021-03-11T21:01:48.376916", "status": "completed"} tags=[]
submission = pd.read_csv('../input/tabular-playground-series-mar-2021/sample_submission.csv')
new_data = pd.read_csv('../input/tabular-playground-series-mar-2021/test.csv')
df = pd.read_csv('../input/tabular-playground-series-mar-2021/train.csv')

# + papermill={"duration": 0.024796, "end_time": "2021-03-11T21:01:52.019798", "exception": false, "start_time": "2021-03-11T21:01:51.995002", "status": "completed"} tags=[]
#profile = df.profile_report(title="Profile train.csv", explorative=True)
#profile.to_file(output_file="profile_report.html")

# + papermill={"duration": 0.116973, "end_time": "2021-03-11T21:01:52.155339", "exception": false, "start_time": "2021-03-11T21:01:52.038366", "status": "completed"} tags=[]
df.drop(columns = "id", inplace = True)
new_data.drop(columns = "id", inplace = True)

# + papermill={"duration": 0.77179, "end_time": "2021-03-11T21:01:52.944501", "exception": false, "start_time": "2021-03-11T21:01:52.172711", "status": "completed"} tags=[]
for col in df.columns[df.dtypes == "object"].tolist():
    df[col] = df[col].astype('category')

# + [markdown] papermill={"duration": 0.016985, "end_time": "2021-03-11T21:01:52.979311", "exception": false, "start_time": "2021-03-11T21:01:52.962326", "status": "completed"} tags=[]
# # Prepare Data

# + papermill={"duration": 0.261591, "end_time": "2021-03-11T21:01:53.258324", "exception": false, "start_time": "2021-03-11T21:01:52.996733", "status": "completed"} tags=[]
X = df.drop('target', axis=1)
y = df['target']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)

# + papermill={"duration": 0.026569, "end_time": "2021-03-11T21:01:53.303125", "exception": false, "start_time": "2021-03-11T21:01:53.276556", "status": "completed"} tags=[]
high_cardinality = ["cat5", "cat7", "cat8", "cat10"]

categorical_cols = X.columns[X.dtypes == "category"].tolist()

categorical_cols = list(set(categorical_cols) - set(high_cardinality))

cat_columns_position = [X.columns.tolist().index(x) for x in categorical_cols + high_cardinality]

# + [markdown] papermill={"duration": 0.017072, "end_time": "2021-03-11T21:01:53.337280", "exception": false, "start_time": "2021-03-11T21:01:53.320208", "status": "completed"} tags=[]
# # Baseline Models

# + papermill={"duration": 0.025974, "end_time": "2021-03-11T21:01:53.381148", "exception": false, "start_time": "2021-03-11T21:01:53.355174", "status": "completed"} tags=[]
categorical_transformer = Pipeline(steps=[
   # ('OrdinalEncoder', ce.OrdinalEncoder(cols=categorical_cols + high_cardinality))
    ('OrdinalEncoder', OrdinalEncoder(handle_unknown = "use_encoded_value", unknown_value = 999))
    #('TargetEncoder', ce.TargetEncoder(cols=high_cardinality))
])

preprocessor = ColumnTransformer(
    transformers=[
        ('cat', categorical_transformer, categorical_cols + high_cardinality)
])

undersample = RandomUnderSampler(sampling_strategy='majority')

# + papermill={"duration": 0.03416, "end_time": "2021-03-11T21:01:53.432523", "exception": false, "start_time": "2021-03-11T21:01:53.398363", "status": "completed"} tags=[]
# Desicion Tree ----------------------------------------
model_tree = DecisionTreeClassifier(random_state = 42)

pipe_tree = Pipeline([('preprocessor', preprocessor),
                      ('clf', model_tree )])

pipe_tree_un = Pipeline([('preprocessor', preprocessor),
                         ('undersample', undersample),
                         ('clf', model_tree )])

# Random Forest ----------------------------------------
model_rf = RandomForestClassifier(random_state = 42, n_jobs=-1)

pipe_rf = Pipeline([('preprocessor', preprocessor),
                    ('clf', model_rf )])

pipe_rf_un = Pipeline([('preprocessor', preprocessor),
                       ('undersample', undersample),
                       ('clf', model_rf )])

# LightGBM ---------------------------------------------
model_lgbm = LGBMClassifier(random_state = 42, 
                            device = "gpu", 
                            n_estimators = 500)

pipe_lgbm = Pipeline([('preprocessor', preprocessor),
                      ('clf', model_lgbm )])

pipe_lgbm_un = Pipeline([('preprocessor', preprocessor),
                         ('undersample', undersample),
                         ('clf', model_lgbm )])

# CatBoost ----------------------------------------------
model_cat = CatBoostClassifier(random_state = 42, verbose = 0,
                               task_type="GPU", border_count = 32,
                                n_estimators = 500,
                                early_stopping_rounds = 100)

pipe_cat = Pipeline([('clf', model_cat )])

pipe_cat_un = Pipeline([('undersample', undersample),
                        ('clf', model_cat )])

# CatBoost (fast mode)----------------------------------
model_cat_fast = CatBoostClassifier(random_seed=42, verbose = 0,
                                    task_type="GPU", border_count = 32, # turn on gpu
                                    n_estimators = 500,
                                     boosting_type = "Plain",
                                     bootstrap_type = "Bernoulli",
                                     max_ctr_complexity = 1,
                                     subsample = 0.6,
                                     #colsample_bylevel = 0.5, # ONLY CPU
                                     #one_hot_max_size = 4,
                                     #leaf_estimation_iterations = 5, #[1,10]
                                     early_stopping_rounds = 100)

pipe_cat_fast = Pipeline([('clf', model_cat_fast )])

pipe_cat_fast_un = Pipeline([('undersample', undersample),
                             ('clf', model_cat_fast )])

# + papermill={"duration": 0.024358, "end_time": "2021-03-11T21:01:53.474052", "exception": false, "start_time": "2021-03-11T21:01:53.449694", "status": "completed"} tags=[]
classifiers = {
    "DecisionTreeClassifier": pipe_tree,
    "DecisionTreeClassifier_un": pipe_tree_un,
    "RandomForestClassifier": pipe_rf,
    "RandomForestClassifier_un": pipe_rf_un,
    "LGBMClassifier": pipe_lgbm,
    "LGBMClassifier_un": pipe_lgbm_un,
    "CatBoostClassifier": pipe_cat,
    "CatBoostClassifier_un": pipe_cat_un,
    "CatBoostClassifier_fast": pipe_cat_fast,
    "CatBoostClassifier_fast_un": pipe_cat_fast_un
}

# + papermill={"duration": 0.026225, "end_time": "2021-03-11T21:01:53.517788", "exception": false, "start_time": "2021-03-11T21:01:53.491563", "status": "completed"} tags=[]
results = pd.DataFrame(columns= ["model", "auc", "aucpr", "logloss",  "time"])

# + papermill={"duration": 214.148032, "end_time": "2021-03-11T21:05:27.683262", "exception": false, "start_time": "2021-03-11T21:01:53.535230", "status": "completed"} tags=[]
# %%time

import time

for key, classifier in classifiers.items():
    print("Running", key)
    
    start_time = time.time()
    
    if key.find("LGBMClassifier") != -1:
        classifiers[key] = classifier.fit(X_train, y_train, clf__categorical_feature = cat_columns_position)
    elif key.find("CatBoostClassifier") != -1:
        classifiers[key] = classifier.fit(X_train, y_train, clf__cat_features = cat_columns_position)
    else:
        classifiers[key] = classifier.fit(X_train, y_train)

    end_time = (time.time() - start_time)

    results = results.append(
        pd.concat([
        pd.DataFrame(evalue_model(classifiers[key], y_test, X_test, key), index=[0]),
        pd.DataFrame({"time" : end_time}, index = [0]) ],
        axis=1)
    )

# + papermill={"duration": 0.05476, "end_time": "2021-03-11T21:05:27.769837", "exception": false, "start_time": "2021-03-11T21:05:27.715077", "status": "completed"} tags=[]
results.sort_values(by=['auc'], ascending = False)

# + papermill={"duration": 12.311426, "end_time": "2021-03-11T21:05:40.113993", "exception": false, "start_time": "2021-03-11T21:05:27.802567", "status": "completed"} tags=[]
from sklearn import metrics
import numpy as np
import matplotlib.pyplot as plt

for key, classifier in classifiers.items():
    
    yhat_prob = [x[1] for x in classifier.predict_proba(X_test)]

    fpr, tpr, thresh = metrics.roc_curve(y_true = y_test, y_score = yhat_prob)
    auc = metrics.roc_auc_score(y_test, yhat_prob)
    plt.plot(fpr,tpr,label=key+", auc="+str(round(auc, 4)))
    
plt.legend(loc=0)

# + [markdown] papermill={"duration": 0.023275, "end_time": "2021-03-11T21:05:40.161217", "exception": false, "start_time": "2021-03-11T21:05:40.137942", "status": "completed"} tags=[]
# # Bayes Tunning

# + papermill={"duration": 0.029156, "end_time": "2021-03-11T21:05:40.213892", "exception": false, "start_time": "2021-03-11T21:05:40.184736", "status": "completed"} tags=[]
# eval dataset to lgbm
#X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42, stratify=y_train)

# + papermill={"duration": 0.035915, "end_time": "2021-03-11T21:05:40.273267", "exception": false, "start_time": "2021-03-11T21:05:40.237352", "status": "completed"} tags=[]
hp_space = {
    'undersample': hp.choice(label = 'undersample', options = [True, False]),
    'clf': {
        #'n_estimators': ho_scope.int(hp.quniform('n_estimators',200,600,100)), 

        'max_depth':  ho_scope.int(hp.quniform('max_depth',1,9,1)),
        #'leaf_estimation_iterations':  ho_scope.int(hp.quniform('leaf_estimation_iterations',1,5,1)),
        
        'subsample': hp.uniform('colsample_bytree',0.2,0.7),
        #'colsample_bylevel': hp.uniform('colsample_bylevel',0.2,0.6),
        
        'reg_lambda': hp.loguniform('reg_lambda',np.log(1e-4),np.log(7))
        
        # catboost 
       # 'border_count': hp.choice(label = 'border_count', options = [32, 64 ,128, 254]) # turn off on gpu
        #'ctr_target_border_count': hp.choice(label = 'ctr_target_border_count', options = [1,5,10,20,50,100,200, 250])
    }
}

ho_sample(hp_space)

# + papermill={"duration": 0.030242, "end_time": "2021-03-11T21:05:40.327095", "exception": false, "start_time": "2021-03-11T21:05:40.296853", "status": "completed"} tags=[]
iteracoes = Trials()


# + papermill={"duration": 0.032399, "end_time": "2021-03-11T21:05:40.383866", "exception": false, "start_time": "2021-03-11T21:05:40.351467", "status": "completed"} tags=[]
def instancia_modelo(hiperparametros):
        
    clf = CatBoostClassifier(**hiperparametros['clf'],
                             random_seed=42, verbose = 0,
                             task_type="GPU", border_count = 32, # turn on gpu
                             boosting_type = "Plain",
                             bootstrap_type = "Bernoulli",
                             max_ctr_complexity = 0,
                             n_estimators = 550,
                             early_stopping_rounds = 100)

    if hiperparametros['undersample'] == True:
        undersample = RandomUnderSampler(sampling_strategy='majority')

    else:
        undersample = None
        
    #categorical_transformer = Pipeline(steps=[
    #    ('OrdinalEncoder', ce.OrdinalEncoder(cols=categorical_cols)),
    #    ('JamesSteinEncoder', ce.JamesSteinEncoder(cols=high_cardinality))
    #])

    #preprocessor = ColumnTransformer(
    #    transformers=[
    #        #('num', numerical_transformer, numerical_cols),
    #        ('cat', categorical_transformer, high_cardinality + categorical_cols)
    #])
    
    pipe = Pipeline([#('preprocessor', preprocessor),
                       ('undersample', undersample),
                       ('clf', clf) ])

    return pipe


# + papermill={"duration": 0.031549, "end_time": "2021-03-11T21:05:40.439357", "exception": false, "start_time": "2021-03-11T21:05:40.407808", "status": "completed"} tags=[]
## criando uma função para realizar o treino do modelo
def funcao_para_minimizar(hiperparametros, features, target):
    
    ## criando uma instancia do modelo com a combinação definida de hiperparametros para usar dentro da função
    pipe = instancia_modelo(hiperparametros)
    
    #pipe.named_steps.preprocessor.fit(X_train, y_train)
    #X_val_interim = pipe.named_steps.preprocessor.transform(X_val)
    # Usando dados de validacao
    
    #eval_set = [X_val, y_val]
    
    #fit_params={'clf__early_stopping_rounds': 100, 
    #            'clf__eval_metric': 'auc',
    #            'clf__verbose': False,
    #            'clf__categorical_feature': categorical_cols + high_cardinality,
    #            'clf__eval_set': eval_set}
    
    fit_params={'clf__cat_features':cat_columns_position}
    
    cv = StratifiedKFold(n_splits=5)
    
    ## treinando o modelo com cross-validation
    resultado = cross_val_score(estimator = pipe, 
                                X = features, 
                                y = target, 
                                scoring = "roc_auc",
                                cv = cv, 
                                error_score = "raise",
                                fit_params = fit_params,
                                n_jobs = -1)
    
    ## retornando a metrica da performance do modelo
    return -resultado.mean()


# + papermill={"duration": 13150.489501, "end_time": "2021-03-12T00:44:50.952972", "exception": false, "start_time": "2021-03-11T21:05:40.463471", "status": "completed"} tags=[]
# %%time

## rodando a otimização
otimizacao = fmin(fn = partial(funcao_para_minimizar, features = X_train, target = y_train),
                  space = hp_space, 
                  algo = tpe.suggest,
                  trials = iteracoes,
                  max_evals = int(120), 
                  rstate = np.random.RandomState(42))


# + papermill={"duration": 0.097475, "end_time": "2021-03-12T00:44:51.136398", "exception": false, "start_time": "2021-03-12T00:44:51.038923", "status": "completed"} tags=[]
def extrai_space_eval(hp_space, trial):
    ## desempacota o resultado
    desempacota_trial = space_eval(space = hp_space, 
                                   hp_assignment = {k: v[0] for (k, v) in trial['misc']['vals'].items() if len(v) > 0})
    
    ## retornando o resultado
    return desempacota_trial


# + papermill={"duration": 0.064211, "end_time": "2021-03-12T00:44:51.277748", "exception": false, "start_time": "2021-03-12T00:44:51.213537", "status": "completed"} tags=[]

def desempacota_dicionario(dicionario):
    desempacotado = {}
    for (chave, valor) in dicionario.items():
        if isinstance(valor, dict):
            desempacotado = {**desempacotado, **desempacota_dicionario(valor)}
        else:
            desempacotado[chave] = valor
            
    return desempacotado


# + papermill={"duration": 0.122831, "end_time": "2021-03-12T00:44:51.456962", "exception": false, "start_time": "2021-03-12T00:44:51.334131", "status": "completed"} tags=[]
## colocando o historico em um dataframe
historico = pd.DataFrame([desempacota_dicionario(extrai_space_eval(hp_space, x)) for x in iteracoes.trials])

## colocando o AUC como uma das colunas
historico['auc'] = [-x['loss'] for x in iteracoes.results]

# + papermill={"duration": 0.064937, "end_time": "2021-03-12T00:44:51.578376", "exception": false, "start_time": "2021-03-12T00:44:51.513439", "status": "completed"} tags=[]
# hiperparâmetros selecionados pela otimização
hiperparametros_selecionados = space_eval(space = hp_space, hp_assignment = otimizacao)
print('Hiperparâmetros selecionados:\n%s' % hiperparametros_selecionados)

# + papermill={"duration": 1.728731, "end_time": "2021-03-12T00:44:53.363621", "exception": false, "start_time": "2021-03-12T00:44:51.634890", "status": "completed"} tags=[]
import plotly.express as px

historico.loc[:,'undersample'] = historico.loc[:,'undersample']*1


fig = px.parallel_coordinates(historico, color="auc")
fig.show()

# + papermill={"duration": 29.397449, "end_time": "2021-03-12T00:45:22.819838", "exception": false, "start_time": "2021-03-12T00:44:53.422389", "status": "completed"} tags=[]
# %%time

# Local results
modelo_final = instancia_modelo(hiperparametros=hiperparametros_selecionados)
modelo_final = modelo_final.fit(X_train, y_train, clf__cat_features=cat_columns_position)

results = results.append(pd.DataFrame(evalue_model(modelo_final, y_test, X_test, "final_model"), index=[0]))
results.sort_values(by=['auc'], ascending = False)

# + papermill={"duration": 273.711746, "end_time": "2021-03-12T00:49:56.623937", "exception": false, "start_time": "2021-03-12T00:45:22.912191", "status": "completed"} tags=[]
# %%time

# Submit results
clf = CatBoostClassifier(**hiperparametros_selecionados['clf'],
                         random_seed=42, verbose = 0, task_type="GPU", border_count = 32,
                         bootstrap_type = "Bernoulli",
                         n_estimators = 3000,
                         early_stopping_rounds = 100)
    
pipe = Pipeline([#('preprocessor', preprocessor),
                 ('undersample', undersample),
                 ('clf', clf) ])

final_fit = pipe.fit(X, y, clf__cat_features=cat_columns_position)

submission.loc[:, 'target'] = final_fit.predict_proba(new_data)[:,1]
submission.to_csv('submission.csv', index = False)

# + papermill={"duration": 0.05896, "end_time": "2021-03-12T00:49:56.742969", "exception": false, "start_time": "2021-03-12T00:49:56.684009", "status": "completed"} tags=[]


StarcoderdataJupyterScriptsDedupFiltered