# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Project Title: E-commerce Analytics ML Hackathon

# #### Author: <NAME>   
# #### Date: 10 April 2020

# With the evolution of the information and communication technologies and the rapid growth of the Internet for the exchange and distribution of information, Electronic Commerce (e-commerce) has gained massive momentum globally, and attracted more and more worldwide users overcoming the time constraints and distance barriers.
#
# It is important to gain in-depth insights into e-commerce via data-driven analytics and identify the factors affecting product sales, the impact of characteristics of customers on their purchase habits.
#
# It is quite useful to understand the demand, habits, concern, perception, and interest of customers from the clue of genders for e-commerce companies. 
#
# However, the genders of users are in general unavailable in e-commerce platforms. To address this gap the aim here is to predict the gender of e-commerce’s participants from their product viewing records.

# **Train file**: CSV containing the product viewing data with gender as label. Product list contains list of products viewed by the user in the given session and it also contains the category, sub category, sub-sub category and the product all encoded and separated with a slash symbol. Each consecutive product is separated with a semicolon.    
#
# **Test file**: CSV containing sessions for which gender prediction is to be submitted. 

# **Evaluation**" Submissions are evaluated on accuracy between the predicted and observed gender for the sessions in the test set.

# ## Import Libraries

# +
import pandas as pd
import numpy as np
from datetime import datetime
import re
import missingno

# visualization
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
import matplotlib.pylab as plt
# %matplotlib inline
from matplotlib.pylab import rcParams

# model
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.model_selection import ShuffleSplit
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier,ExtraTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import AdaBoostClassifier,RandomForestClassifier,StackingClassifier,VotingClassifier,BaggingClassifier,ExtraTreesClassifier,GradientBoostingClassifier,BaggingClassifier,ExtraTreesClassifier
from sklearn.model_selection import GridSearchCV, KFold
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.ensemble import GradientBoostingClassifier
from xgboost.sklearn import XGBClassifier
import xgboost as xgb
import lightgbm as lgb
import catboost as cat

# evaluation
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.ensemble import VotingClassifier
from sklearn.metrics import f1_score, roc_auc_score, accuracy_score

from sklearn.utils import resample
from sklearn.preprocessing import LabelEncoder
import warnings
warnings.simplefilter('ignore')
# -

pd.set_option("display.max_columns", None)
pd.set_option("display.max_rows", None)
pd.set_option("display.max_colwidth", -1)

# ## Import Data

train = pd.read_csv("train.csv")
test = pd.read_csv("test.csv")

train.info()

train.head()

test.info()

test.head()

# ## Check for Missing Values

# check for missing values
miss = train.isnull().sum()/len(train)
miss = miss[miss > 0]
miss.sort_values(inplace=True)
miss


# ## Data Preprocessing

# +
def preprocess(data):
    # convert starttime and endtime to datetime format
    data["startTime"] = pd.to_datetime(data["startTime"])
    data["endTime"] = pd.to_datetime(data["endTime"])
    print("Done: Convert to Datetime format")
    
    # get the number of sites visited
    data["total_products"] = data["ProductList"].apply(lambda x: len(x.split(";")))
    print("Done: Number of sites visited")
    
#     # extract product D group
#     data["D_group"] = data["ProductList"].apply(lambda x: re.search(r"D\d{3}", x).group(0))
#     print("Done: Extracting D Group")
    
    # extract the productlist
    temp = data["ProductList"].str.split(";")
    data = data.reindex(data.index.repeat(temp.apply(len)))
    data["product_data"] = np.hstack(temp)
    data['category'] = data["product_data"].str.split("/").str[0]
    data["sub_category"] = data["product_data"].str.split("/").str[1]
    data["sub_sub_category"] = data["product_data"].str.split("/").str[2]
    data["product"] = data["product_data"].str.split("/").str[3]
    #data["ProductList"] = data["ProductList"].apply(lambda x: x.replace(";",""))
    #data["ProductList"] = data["ProductList"].apply(lambda x: re.sub("\/D\d+", "", x))
#     columns = sorted(set(data['ProductList'].str.split('/').sum()))
#     for column in columns:
#         data[column] = data["ProductList"].str.count(column)
    print("Done: Extracting categories")
    
    # extract duration spent on website
    duration = data["endTime"] - data["startTime"]
    data["duration"] = duration.dt.seconds
    data["startTime_month"] = data["startTime"].dt.month
    data["startTime_date"] = data["startTime"].dt.date
    data["startTime_day"] = data["startTime"].dt.day
    data["startTime_doy"] = data["startTime"].dt.dayofyear
    data["startTime_week"] = data["startTime"].dt.week
    data["startTime_dow"] = data["startTime"].dt.dayofweek
    data["startTime_hour"] = data["startTime"].dt.hour
    data["startTime_minute"] = data["startTime"].dt.minute
    data["endTime_hour"] = data["endTime"].dt.hour
    data["endTime_minute"] = data["endTime"].dt.minute
    print("Done: Extracting duration spent on website")
    
    # total
    data["total_product_day"] = data.groupby(["startTime_day"])["total_products"].transform("sum")
    data["total_product_week"] = data.groupby(["startTime_week"])["total_products"].transform("sum")
    data["total_product_dow"] = data.groupby(["startTime_dow"])["total_products"].transform("sum")
    data["total_product_hour"] = data.groupby(["startTime_hour"])["total_products"].transform("sum")
    print("Done: Extracting total information")
                                              
    return data


# -

train = preprocess(train)

test = preprocess(test)

# remove gender from training set first
train_df = train.drop("gender", axis=1)
y = train["gender"]

# ## Encoding Categorical

le_cat = LabelEncoder()
le_subcat = LabelEncoder()
le_subsubcat = LabelEncoder()
le_product = LabelEncoder()
le_gender = LabelEncoder()
le_session = LabelEncoder()

combined_data = train_df.append(test)

# fit on the combined data
combined_data["category"] = le_cat.fit_transform(combined_data["category"])
combined_data["sub_category"] = le_subcat.fit_transform(combined_data["sub_category"])
combined_data["sub_sub_category"] = le_subsubcat.fit_transform(combined_data["sub_sub_category"])
combined_data["product"] = le_product.fit_transform(combined_data["product"])
combined_data["session_id"] = le_session.fit_transform(combined_data["session_id"])
y = le_gender.fit_transform(y)

# apply transformation (train)
train_df["category"] = le_cat.transform(train_df["category"])
train_df["sub_category"] = le_subcat.transform(train_df["sub_category"])
train_df["sub_sub_category"] = le_subsubcat.transform(train_df["sub_sub_category"])
train_df["product"] = le_product.transform(train_df["product"])
train_df["session_id"] = le_session.transform(train_df["session_id"])

# apply transformation (test)
test["category"] = le_cat.transform(test["category"])
test["sub_category"] = le_subcat.transform(test["sub_category"])
test["sub_sub_category"] = le_subsubcat.transform(test["sub_sub_category"])
test["product"] = le_product.transform(test["product"])
test["session_id"] = le_session.transform(test["session_id"])

# ## Remove unnecessary columns

train_df = train_df.drop(["startTime", "endTime", "ProductList", "product_data", "startTime_date"], axis=1)
test = test.drop(["startTime", "endTime", "ProductList", "product_data", "startTime_date"], axis=1)


# ## SMOTE

# +
# # concatenate our training data back together
# X = pd.concat([X_train, y_train], axis=1)

# # separate minority and majority classes
# male = X[X["gender"] == "male"]
# female = X[X["gender"] == "female"]

# # upsample minority
# male_upsampled = resample(male,
#                           replace = True, # sample with replacement
#                           n_samples = len(female), # match number in majority class)
#                           random_state = 101)

# # combine majority and upsample minority
# upsampled = pd.concat([female, male_upsampled])

# # check new class counts
# upsampled.gender.value_counts()

# +
# X_train = upsampled.drop("gender", axis=1)
# y_train = upsampled["gender"]
# -

# ## Match Test Set Columns with Training Set Columns

# +
# # get missing columns in training and test set
# missing_cols = set(train.columns) - set(test.columns)

# # add a missing column in test set with default value equal to 0
# for c in missing_cols:
#     test[c] = 0
    
# # ensure that order of column in the test set is in the same order than in train set
# test = test[train.columns]

# +
# test = test.drop("gender", axis=1)
# -

# ## Baseline Performance

def baseliner(features, target, cv=3, metric="accuracy"):
    print("Baseline Models\n")
    eval_dict = {}
    models = [lgb.LGBMClassifier(), xgb.XGBClassifier(), cat.CatBoostClassifier(verbose=0), GradientBoostingClassifier(), 
              LogisticRegression(), RandomForestClassifier(), DecisionTreeClassifier(), AdaBoostClassifier(),
              ExtraTreeClassifier(), ExtraTreesClassifier(), KNeighborsClassifier(), BaggingClassifier()]
    print("Model Name \t |    CV")
    print("--"*50)
    
    for index, model in enumerate(models, 0):
        model_name = str(model).split("(")[0]
        eval_dict[model_name] = {}
        
        results = cross_val_score(model, features, target, cv=cv, scoring=metric)
        eval_dict[model_name]["cv"] = results.mean()
        
        print("%s \t | %.4f \t" % (model_name[:12], eval_dict[model_name]["cv"]))


baseliner(train_df, y, cv=5, metric="accuracy")

# ## Training and Testing Set

rs = ShuffleSplit(n_splits=3, test_size=0.1, random_state=101)
test_prob_preds = np.zeros(test.shape[0])

# ## XGBoost Classifier

# +
for idx, (train_idx, valid_idx) in enumerate(rs.split(train_df, pd.DataFrame(y))):
    print("--"*40)
    print("Iteration Number : {}".format(idx))
    MAX_ROUNDS=2000
    early_stopping_rounds=100
    params = {
        'booster': 'gbtree',
        'objective': 'binary:logistic',
        'eval_metric': 'error',
        'learning_rate': 0.1,
        'num_round': MAX_ROUNDS,
        'max_depth': 8,
        'seed': 13,
        'nthread': -1
    }

    X_train, X_valid, y_train, y_valid = train_df.iloc[train_idx], train_df.iloc[valid_idx], pd.DataFrame(y).iloc[train_idx], pd.DataFrame(y).iloc[valid_idx]
    
    dtrain = xgb.DMatrix(X_train, label=y_train)
    dvalid = xgb.DMatrix(X_valid, label=y_valid)
    watchlist = [(dtrain, 'train'), (dvalid, 'valid')]
    
    model = xgb.train(
        params,
        dtrain,
        evals=watchlist,
        num_boost_round=MAX_ROUNDS,
        early_stopping_rounds=early_stopping_rounds,
        verbose_eval=50
    )
    
    print("Best Iteration :: {} \n".format(model.best_iteration))

    # Plotting Importances
    fig, ax = plt.subplots(figsize=(24, 24))
    xgb.plot_importance(model, height=0.4, ax=ax)
    preds = model.predict(xgb.DMatrix(test), ntree_limit=model.best_ntree_limit)
    
    test_prob_preds += preds
    
test_prob_preds /= rs.n_splits
print(test_prob_preds.shape, test_prob_preds[: 5])
# -

test.head()

# take the mean of the proabilities for all the session_ids to calculate male or female
test["predict_prob"] = test_prob_preds
test["session_id"] = le_session.inverse_transform(test["session_id"])
agg = test.groupby(["session_id"])["predict_prob"].mean().reset_index()

# ## Submission

sub = pd.read_csv("sample_submission.csv")
sub = sub.drop("gender", axis=1)
sub = pd.merge(sub, agg, on=["session_id"], how='left')

sub["gender"] = sub["predict_prob"].apply(lambda x: round(x))

sub["gender"] = sub["gender"].map({1: "male", 0: "female"})

sub = sub.drop("predict_prob", axis=1)

sub.to_submission("submission.csv")

StarcoderdataJupyterScriptsDedupFiltered