# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# ## Try Featuretools - Auto Basic Feature Engineering
#
# * It does basic feature engineering, by generating MIN, MAX, SUM, STD, SKEW, NUM_UNIQUE, MEAN, MODE.
# * About Featuretools: https://github.com/Featuretools/featuretools
#
#
# * Data can be downloaded here (need sign in): https://datahack.analyticsvidhya.com/contest/practice-problem-big-mart-sales-iii/
# * Reference: https://www.analyticsvidhya.com/blog/2018/08/guide-automated-feature-engineering-featuretools-python/?utm_source=feedburner&utm_medium=email&utm_campaign=Feed%3A+AnalyticsVidhya+%28Analytics+Vidhya%29

import featuretools as ft
import numpy as np
import pandas as pd

train = pd.read_csv("Big_Mart_Train.csv")
train.head()

# +
# Data Preprocessing

train.isnull().sum()
# -

train.Item_Weight = train.Item_Weight.fillna(np.nanmedian(train.Item_Weight))

print train.Outlet_Size.unique()

# fill NA with mode
train.Outlet_Size = train.Outlet_Size.fillna(train.Outlet_Size.mode().iloc[0])

train.dtypes

print train.Item_Fat_Content.unique()
print train.Item_Type.unique()
print train.Outlet_Identifier.unique()
print train.Outlet_Size.unique()
print train.Outlet_Location_Type.unique()
print train.Outlet_Type.unique()

train.Item_Fat_Content = train.Item_Fat_Content.replace(['low fat', 'LF'], ['Low Fat', 'Low Fat'])
train.Item_Fat_Content = train.Item_Fat_Content.replace(['reg'], ['Regular'])

print train.Item_Fat_Content.unique()

train.head()

train2 = train.copy(deep=True)  # make a copy of the dataframe, not the reference, changes won't change this copy

# +
# Featuretools allows you to create multiple tables
train['id'] = train['Item_Identifier'] + train['Outlet_Identifier']  # id used for index
# train.drop(['Item_Identifier'], axis=1, inplace=True)

## step 1 - create an entity set, it could contain multiple tables and the relationships between tables
es = ft.EntitySet(id = 'sales')

## step 2 - create base entity
es.entity_from_dataframe(entity_id = 'bigmart', dataframe = train, index = 'id')

## step 3 - create 2 seperate tables
es.normalize_entity(base_entity_id='bigmart', new_entity_id='outlet', index = 'Outlet_Identifier', 
additional_variables = ['Outlet_Establishment_Year', 'Outlet_Size', 'Outlet_Location_Type', 'Outlet_Type'])

es.normalize_entity(base_entity_id='bigmart', new_entity_id='item', index = 'Item_Identifier', 
additional_variables = ['Item_Weight', 'Item_Fat_Content', 'Item_Visibility', 'Item_Type', 'Item_MRP'])
# -

feature_matrix, feature_names = ft.dfs(entityset=es, 
target_entity = 'outlet', 
max_depth = 2, 
verbose = 1, 
n_jobs = 3)

feature_matrix.head()

# correct the index
feature_matrix = feature_matrix.reindex(index=train['Outlet_Identifier'])
feature_matrix = feature_matrix.reset_index()

feature_matrix.head()

feature_matrix.columns

# ## Summary
#
# * When I chose the target_entity as "outlet", its additional variables won't go through feature engineering. But other columns in the base entity will all be processed.
#   * So it seems that "additional variables" are just to specify those columns in the target entity that won't be processed.
# * For some reason, when I tried to process "item" as the target entity, always get many errors.
# * `max_depth` is 2 is enough for the columns here, if at level 2 there will be columns that can be processed further, higher depth will be good.

StarcoderdataJupyterScriptsDedupFiltered