# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
#import janitor as jn

pd.set_option('max_columns', None)

# ## Charger les données

data = pd.read_pickle("../../data/processed/maison.pkl")
#data.to_pickle("../../data/processed/maison.pkl")

# Afficher la table initiale
print(data.info(null_counts=True))

# ## Traitement des données

num_features = data.select_dtypes("float").columns
categ_features = data.select_dtypes("category").columns

data[categ_features] = data[categ_features].astype("str")
data[num_features] = data[num_features].astype("float32")
#data[["surface_terrain","valeur_fonciere"]] = data[["surface_terrain","valeur_fonciere"]].astype("float32")

data.replace([np.inf, -np.inf], np.nan).dropna(how="any",inplace=True)

data.describe()

data.info()

# +
#data["code_postal"] = data["code_postal"].astype("str")
cat_f_to_keep = ["code_postal"]
cat_f_to_ignore = categ_features[categ_features!=cat_f_to_keep[0]]

target = 'valeur_fonciere'

num_f_to_keep = list(num_features[num_features!=target])

# +
sub_pc = (data["code_postal"].value_counts().cumsum()<50000)
list_sub_pc = list(sub_pc.index[sub_pc])
df = data[data["code_postal"].isin(list_sub_pc)]

#df = df[(df["valeur_fonciere"]>30000) & (df["valeur_fonciere"]<300000)]
print(data.shape, df.shape, sub_pc.sum())
# -

df["surface_terrain"].hist()

#len(list_sub_pc)
sub_pc.index[1]

data["code_postal"].isin(list_sub_pc).sum()

pd.DataFrame([data["code_postal"].value_counts(), data["code_postal"].value_counts().cumsum()]).T

allo[444]

df["code_postal"].value_counts().shape

df = data[(data["valeur_fonciere"]>30000) & (data["valeur_fonciere"]<300000)]
print(data.shape, df.shape)

# ## Pycaret - Regression

# PROB : pour la scission des données, faire proportionnel dans chaque classe. Mais il y a peut-etre des classes manquantes dans les données d'entrainement que l'on a seulement dans l'ensemble de test.

from pycaret.regression import *

exp_reg101 = setup(data = df, target = target, ignore_features=cat_f_to_ignore, categorical_features=cat_f_to_keep,
                   numeric_features=num_f_to_keep,
                   remove_outliers=True, #remove_multicollinearity = True,# multicollinearity_threshold = 0.6,
                   #combine_rare_levels = True, rare_level_threshold=0.2,
                   feature_selection=True,
                   polynomial_features = True,
                   trigonometry_features = True,
                   feature_interaction=True,
                   feature_ratio=True,
                   create_clusters=True,
                   use_gpu=True,
                   normalize = True, normalize_method="robust", transformation = True, transform_target = True, verbose=True, session_id=123)
# normalize_method='robust'

# +
# exp_reg101 = setup(data = data, target = 'valeur_fonciere', remove_outliers=True,
#                    ignore_features=cat_features, create_clusters=True, polynomial_features=True, feature_ratio=True, 
#                    profile=True, session_id=123) 
# exp_reg101 = setup(data = data, target = target, remove_outliers=True, categorical_features=cat_f_tokeep,
#                    numeric_features=num_f_tokeep, ignore_features=cat_f_toignore,
#                    session_id=123) 
# -

compare_models(exclude=["catboost", "svm", "et", "tr", "rf", "ransac", "gbr","ada"], round=4, fold=5, n_select=3)

llar = create_model("llar")
#mlp = create_model("mlp")

tune_llar = tune_model(llar, n_iter=1000)

evaluate_model(tune_llar)

tune_mlp = tune_model(mlp, n_iter=100)

pm = predict_model(tune_llar)

pm.sort_values(by="Label",ascending=False)

models()

# ## Un seul code postal

categ_features

data["adresse_code_voie"]



# +
#data["code_postal"] = data["code_postal"].astype("str")
cat_f_to_keep = ["adresse_code_voie"]
cat_f_to_ignore = ["code_postal","code_commune"]

target = 'valeur_fonciere'

num_f_to_keep = list(num_features[num_features!=target])

sub_pc = data["code_postal"].value_counts()
list_sub_pc = sub_pc.index[0]
print(list_sub_pc)
df2 = data[data["code_postal"].isin([list_sub_pc])]

df2 = df2[(df2["valeur_fonciere"]>30000) & (df2["valeur_fonciere"]<300000)]
print(data.shape, df2.shape, sub_pc.sum(), list_sub_pc)
# -

exp_un_1 = setup(data = df2, target = target, ignore_features=cat_f_to_ignore, categorical_features=cat_f_to_keep,
                   numeric_features=num_f_to_keep,
                   remove_outliers=True, #remove_multicollinearity = True,# multicollinearity_threshold = 0.6,
                   #combine_rare_levels = True, rare_level_threshold=0.2,
                   feature_selection=True,
                   polynomial_features = True,
                   trigonometry_features = True,
                   feature_interaction=True,
                   feature_ratio=True,
                   create_clusters=True,
                   use_gpu=True,
                   normalize = True, normalize_method="robust", transformation = True, transform_target = True, verbose=True, session_id=123)
# normalize_method='robust'

compare_models(round=4, fold=5, n_select=3)

gbr = create_model("gbr")

tuned_gbr = tune_model(gbr, n_iter)

evaluate_model(tuned_gbr)

# ## Restes

interpret_model(xg_reg)

xg_reg = automl(use_holdout=False)
xg_reg

plot_model(xg_reg, plot = 'residuals')

pull()

xg_reg = create_model("xgboost")
tune_xg_reg = tune_model(xg_reg, n_iter=100, fold=10, choose_better=True)
print(tune_xg_reg)

tune_xg_reg2 = tune_model(tune_xg_reg2, n_iter=10, fold=10, choose_better=True)
print(tune_xg_reg)

print(tune_xg_reg2)

finalize_model(xg_reg)

save_model(xg_reg,'../../models/final_xg_reg_for_streamlit_test')

predict_model(xg_reg)

interpret_model(xg_reg)



rf_reg = create_model("rf")
tune_rf_reg = tune_model(rf_reg, n_iter=100, fold=10, choose_better=True)
print(tune_rf_reg)

compare_models(whitelist=["catboost", "svm"], fold=5, n_select=2)

compare_models(blacklist=["catboost"], fold=5, n_select=1)

catboost_reg = create_model("catboost")



tune_huber_reg = tune_model(huber_reg, n_iter=100, fold=10, choose_better=True)
print(tune_huber_reg)

# +
#evaluate_model(tune_huber_reg)
# -

interpret_model(tune_huber_reg)

exp_reg102 = setup(data = data, target = 'valeur_fonciere', remove_outliers=True,
                   ignore_features=cat_features, polynomial_features=True,
                   session_id=124) 

compare_models(blacklist=["catboost","svm"], fold=5, n_select=1)

lr_reg = create_model("lr")

tune_lr_reg = tune_model(lr_reg, n_iter=100, fold=10, choose_better=True)
print(tune_lr_reg)

interpret_model(tune_lr_reg)

blend_all = blend_models()

data["code_postal"] = data["code_postal"].apply(lambda x: f"{x}")

data["code_postal"]

pd.Series.apply()

StarcoderdataJupyterScriptsDedupFiltered