<jupyter_start><jupyter_text>XGBoost<jupyter_code>from sklearn.model_selection import cross_val_score, KFold
from sklearn.metrics import roc_auc_score
from bayes_opt import BayesianOptimization
from IPython.display import display
from ipywidgets import IntProgress
from sklearn import metrics
import xgboost as xgb
import pandas as pd
import os
if '__file__' in locals():
    current_folder = os.path.dirname(os.path.abspath(__file__))
else:
    current_folder = os.getcwd()

set_de_entrenamiento_testing_y_prediccion = '"{}"'.format(os.path.join(
    current_folder,
    '..',
    'Set de entrenamiento, testing y predicción.ipynb'
))
merge_features = '"{}"'.format(os.path.join(current_folder, '..', 'Features', 'Merge features.ipynb'))
calcular_auc = '"{}"'.format(os.path.join(current_folder, '..', 'Calcular AUC.ipynb'))
predicciones_csv = os.path.join(current_folder, '..', 'predictions.csv')
hiperparametros_csv = os.path.join(current_folder, 'hiperparametros', 'xgboost.csv')<jupyter_output><empty_output><jupyter_text>Cargo el df con los features.<jupyter_code>pd.options.mode.chained_assignment = None
%run $merge_features
assert(df_features.shape[0] == df['person'].unique().shape[0])<jupyter_output><empty_output><jupyter_text>Cargo los sets de entrenamiento, testing y predicción.<jupyter_code>%run $set_de_entrenamiento_testing_y_prediccion

labels_with_features = labels.merge(df_features, how='inner', on='person')
data = labels_with_features.drop('label', axis=1)
target = labels_with_features['label']<jupyter_output><empty_output><jupyter_text>Entrenamiento rápido Con cross validation de xgboost. Lo bueno de esto es que al final me da el *num_boost_round* óptimo.<jupyter_code>param = {
    'silent': 1,
    'objective': 'reg:logistic',
    'alpha': 3.845311207046479,
    'colsample_bylevel': 0.6605668347627213,
    'colsample_bytree': 0.5279014819087092,
    'eta': 0.15803667962605694,
    'gamma': 6.219264874528072,
    'lambda': 1.1181195507921775,
    'max_delta_step': 7.592652591386328,
    'max_depth': 9,
    'min_child_weight': 4.302125582335056,
    'subsample': 0.43744176565530823
}
cv = 10 # cantidad de splits en el cross validation
num_round = 100 # cantidad de veces que se boostea
%%time
dtrain = xgb.DMatrix(data, label=target)
result = xgb.cv(param, dtrain, nfold=cv, metrics='auc', verbose_eval=False, shuffle=False, stratified=False, num_boost_round=num_round)<jupyter_output>/home/sebas/.envs/trocafone/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version
  if getattr(data, 'base', None) is not None and \
/home/sebas/.envs/trocafone/lib/python3.6/site-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version
  data.base is not None and isinstance(data, np.ndarray) \<jupyter_text>El índice + 1 es el *num_boost_round* óptimo.<jupyter_code>result.loc[[result['test-auc-mean'].idxmax()]]<jupyter_output><empty_output><jupyter_text>Búsqueda de hiperparámetros con Grid SearchVamos a hacer un kfold con sklearn.**Nota**: está busqueda ya no es óptima, es mejor realizarla usando GridSearchCV de sklearn o Bayesian Optimization que está más abajo.<jupyter_code>splits = 2
max_depth_values = 2
eta_values = 2
gamma_values = 2
num_round_values = 2
param = {
    'max_depth': 30,
    'eta': 0,
    'gamma': 0,
    'silent': 1,
    'objective': 'binary:logistic',
    'nthread': 10,
    'eval_metric': 'auc'
}
def calculate_auc():
    return metrics.roc_auc_score(labels_test['label'], labels_test['label_predicted'])
labels_with_features = labels.merge(df_features, how='inner', on='person')
columns = list(labels_with_features.columns)
columns.remove('label')
f = IntProgress(min=0, max=splits*max_depth_values*eta_values*gamma_values*num_round_values)
display(f) # display the bar

kf = KFold(n_splits=splits, shuffle=False)
results = pd.DataFrame(columns=['k', 'max_depth', 'eta', 'gamma', 'num_round', 'auc'])
index = 0
k = 0
for train_index, test_index in kf.split(labels):
    
    labels_training = labels_with_features.iloc[train_index]
    labels_test = labels_with_features.iloc[test_index]
    assert(labels_training.merge(labels_test, how='inner', on='person').shape[0] == 0)
    train_matrix = xgb.DMatrix(labels_training.loc[:, columns], label=labels_training['label'])
    test_matrix = xgb.DMatrix(labels_test.loc[:, columns])
    
    for max_depth, eta, gamma, num_round in np.ndindex((max_depth_values, eta_values, gamma_values, num_round_values)):
        eta=eta/eta_values
        param['max_depth'] = max_depth
        param['eta'] = eta
        param['gamma'] = gamma
        
        bst = xgb.train(param, train_matrix, num_round)
        labels_test['label_predicted'] = bst.predict(test_matrix)
        
        results.loc[index] = k, max_depth, eta, gamma, num_round, calculate_auc()
        
        index+=1
        f.value += 1
    
    k += 1<jupyter_output><empty_output><jupyter_text>Promedio los hiperparámetros.<jupyter_code>results_mean = results.groupby(['max_depth', 'eta', 'gamma', 'num_round'])[['auc']].mean()
mejor_resultado = results_mean.loc[results_mean.idxmax()]
mejor_resultado<jupyter_output><empty_output><jupyter_text>Escribo los nuevos resultados en un archivo.<jupyter_code>params = mejor_resultado.reset_index().to_dict('records')[0]
auc = params.pop('auc')
hyperparameter_data = {
    'algorithm': 'xgboost',
    'hyperparameters': params,
    'cv_splits': splits,
    'auc': auc,
    'features': data.columns
}
%run -i write_hyperparameters.py<jupyter_output><empty_output><jupyter_text>Hiperparámetros con Bayesian Optimization<jupyter_code>pbounds = {
    'max_depth': (2, 30),
    'eta': (0, 1),
    'gamma': (0, 20),
    'min_child_weight': (1, 8),
    'max_delta_step': (1, 8),
    'subsample': (0, 1),
    'colsample_bytree': (0, 1),
    'colsample_bylevel': (0, 1),
    'lambda': (1, 10),
    'alpha': (0, 8)
}

discrete = ['max_depth'] # parámetros discretos
cv_splits = 10 # cantidad de splits en el cv
num_round = 100 # cantidad máxima de boosts<jupyter_output><empty_output><jupyter_text>Falta optimizar otros parámetros discretos: - booster  - min_child_weight  - max_delta_step  - etc...<jupyter_code>dtrain = xgb.DMatrix(data, label=target)
def cv_score_xgb(**param):
    param['silent'] = 1
    param['objective'] = 'reg:logistic'
#     param['scale_pos_weight'] = 19
    
    # transformo los valores que deben ser discretos
    for d in discrete:
        param[d] = int(param[d])
    
    # hago el cv
    scores = xgb.cv(param, dtrain, nfold=cv_splits, metrics='auc', verbose_eval=False, shuffle=False, stratified=False, num_boost_round=num_round, early_stopping_rounds=20)
    return scores['test-auc-mean'].max()
%%time
optimizer = BayesianOptimization(f=cv_score_xgb, pbounds=pbounds)
optimizer.probe(
    params = {
        'alpha': 3.845311207046479,
        'colsample_bylevel': 0.6605668347627213,
        'colsample_bytree': 0.5279014819087092,
        'eta': 0.15803667962605694,
        'gamma': 6.219264874528072,
        'lambda': 1.1181195507921775,
        'max_delta_step': 7.592652591386328,
        'max_depth': 9,
        'min_child_weight': 4.302125582335056,
        'subsample': 0.43744176565530823
    }
)
    
optimizer.maximize(
    init_points=0,
    n_iter=100,
)
optimizer.max<jupyter_output><empty_output><jupyter_text>Guardo el resultado en un archivo.<jupyter_code>params = optimizer.max['params'].copy()
params['max_depth'] = int(params['max_depth'])
params['silent'] = 1
result = xgb.cv(params, dtrain, nfold=cv_splits, metrics='auc', verbose_eval=False, shuffle=False, stratified=False, num_boost_round=num_round)
params['num_round'] = result['test-auc-mean'].idxmax() + 1
del params['silent']
hyperparameter_data = {
    'algorithm': 'xgboost',
    'hyperparameters': params,
    'cv_splits': cv_splits,
    'auc': optimizer.max['target'],
    'features': data.columns
} 
%run -i write_hyperparameters.py<jupyter_output><empty_output><jupyter_text>Predecir labels desconocidos<jupyter_code>dtrain = xgb.DMatrix(data, label=target)
param = {
    'silent': 1,
    'objective': 'reg:logistic',
    'alpha': 3.845311207046479,
    'colsample_bylevel': 0.6605668347627213,
    'colsample_bytree': 0.5279014819087092,
    'eta': 0.15803667962605694,
    'gamma': 6.219264874528072,
    'lambda': 1.1181195507921775,
    'max_delta_step': 7.592652591386328,
    'max_depth': 9,
    'min_child_weight': 4.302125582335056,
    'subsample': 0.43744176565530823 
}
num_round = 10
bst = xgb.train(param, dtrain, num_boost_round=65)<jupyter_output><empty_output><jupyter_text>Predigo:<jupyter_code>labels_to_predict_with_features = labels_to_predict.merge(df_features, how='inner', on='person')
assert(labels_to_predict.shape[0] == labels_to_predict_with_features.shape[0])
matrix = xgb.DMatrix(labels_to_predict_with_features.loc[:, columns])
labels_to_predict['label'] = bst.predict(matrix)
labels_to_predict.to_csv(predicciones_csv)<jupyter_output><empty_output>