# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# + [markdown] heading_collapsed=true
# # 0.0. IMPORTS

# + hidden=true
import math
import numpy as np
import pandas as pd
import inflection
import datetime
import seaborn as sns

from scipy                import stats as ss
from tabulate             import tabulate
from matplotlib           import pyplot as plt
from IPython.display      import Image
from IPython.core.display import HTML

from sklearn.preprocessing import RobustScaler, MinMaxScaler, LabelEncoder


# + [markdown] heading_collapsed=true hidden=true
# ## 0.1. Helper Functions

# + hidden=true

def cramer_v( x, y ):

    cm = pd.crosstab( x, y ).values
    n = cm.sum()
    r, k = cm.shape

    chi2 = ss.chi2_contingency( cm )[0]
    chi2cor = max( 0, chi2 - (k-1)*(r-1)/(n-1))

    rcor = r - ((r-1)**2)/(n-1)
    kcor = k - ((k-1)**2)/(n-1)

    v= np.sqrt( ( chi2cor/n ) / ( min( kcor-1, rcor-1 ) ) )
    
    return v

def jupyter_settings():
    # %matplotlib inline
    # %pylab inline
    
    plt.style.use( 'bmh' )
    plt.rcParams['figure.figsize'] = [25, 12]
    plt.rcParams['font.size'] = 24
    
    display( HTML( '<style>.container { width:100% !important; }</style>') )
    pd.options.display.max_columns = None
    pd.options.display.max_rows = None
    pd.set_option( 'display.expand_frame_repr', False )
    
    sns.set()


# + hidden=true
jupyter_settings()


# + [markdown] heading_collapsed=true hidden=true
# ## 0.2. Loading Data

# + hidden=true
df_sales_raw = pd.read_csv( 'data/train.csv', low_memory=False )
df_store_raw = pd.read_csv( 'data/store.csv', low_memory=False )

# merge
df_raw = pd.merge( df_sales_raw, df_store_raw, how='left', on='Store' )

# + [markdown] heading_collapsed=true
# # 1.0. DESCRICAO DOS DADOS

# + hidden=true
df1 = df_raw.copy()

# + [markdown] heading_collapsed=true hidden=true
# ## 1.1. Rename Columns

# + hidden=true
cols_old = ['Store', 'DayOfWeek', 'Date', 'Sales', 'Customers', 'Open', 'Promo', 'StateHoliday', 'SchoolHoliday', 
            'StoreType', 'Assortment', 'CompetitionDistance', 'CompetitionOpenSinceMonth', 'CompetitionOpenSinceYear', 
            'Promo2', 'Promo2SinceWeek', 'Promo2SinceYear', 'PromoInterval']

snakecase = lambda x: inflection.underscore( x )

cols_new = list( map( snakecase, cols_old ) )

# rename
df1.columns = cols_new

# + [markdown] heading_collapsed=true hidden=true
# ## 1.2. Data Dimensions

# + hidden=true
print( 'Number of Rows: {}'.format( df1.shape[0] ) )
print( 'Number of Cols: {}'.format( df1.shape[1] ) )

# + [markdown] heading_collapsed=true hidden=true
# ## 1.3. Data Types

# + hidden=true
df1['date'] = pd.to_datetime( df1['date'] )
df1.dtypes

# + [markdown] heading_collapsed=true hidden=true
# ## 1.4. Check NA

# + hidden=true
df1.isna().sum()

# + [markdown] heading_collapsed=true hidden=true
# ## 1.5. Fillout NA

# + code_folding=[] hidden=true
#competition_distance      
df1['competition_distance'] = df1['competition_distance'].apply( lambda x: 200000.0 if math.isnan( x ) else x )


#competition_open_since_month   
df1['competition_open_since_month'] = df1.apply( lambda x: 
                                                 x['date'].month if math.isnan( x['competition_open_since_month'] ) else 
                                                 x['competition_open_since_month'], axis=1 )

#competition_open_since_year
df1['competition_open_since_year'] = df1.apply( lambda x: 
                                                 x['date'].year if math.isnan( x['competition_open_since_year'] ) else 
                                                 x['competition_open_since_year'], axis=1 )

#promo2_since_week
df1['promo2_since_week'] = df1.apply( lambda x: x['date'].week if math.isnan( x['promo2_since_week'] ) else 
                                                x['promo2_since_week'], axis=1 )

#promo2_since_year
df1['promo2_since_year'] = df1.apply( lambda x: x['date'].year if math.isnan( x['promo2_since_year'] ) else 
                                      x['promo2_since_year'], axis=1 )

#promo_interval
month_map = {1: 'Jan', 2: 'Fev', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec'}

df1['promo_interval'].fillna( 0, inplace=True )

df1['month_map'] = df1['date'].dt.month.map( month_map )

df1['is_promo'] = df1[['promo_interval', 'month_map']].apply( lambda x: 0 if x['promo_interval'] == 0 else
                                                              1 if x['month_map'] in x['promo_interval'].split(',') else 0, axis=1 )

# + hidden=true
df1.sample(5)

# + hidden=true
df1.isna().sum()

# + [markdown] heading_collapsed=true hidden=true
# ## 1.6. Change Data Types

# + hidden=true
# competition
df1['competition_open_since_month'] = df1['competition_open_since_month'].astype( int )
df1['competition_open_since_year'] = df1['competition_open_since_year'].astype( int )

# promo2
df1['promo2_since_week'] = df1['promo2_since_week'].astype( int )
df1['promo2_since_year'] = df1['promo2_since_year'].astype( int )

# + [markdown] heading_collapsed=true hidden=true
# ## 1.7. Descriptive Statistics

# + hidden=true
num_attributes = df1.select_dtypes( include=['int64', 'float64'] )
cat_attributes = df1.select_dtypes( exclude=['int64', 'float64', 'datetime64[ns]'] )

# + [markdown] heading_collapsed=true hidden=true
# ### 1.7.1. Numerical Attributes

# + hidden=true
# Central Tendency - mean, median
ct1 = pd.DataFrame( num_attributes.apply( np.mean ) ).T
ct2 = pd.DataFrame( num_attributes.apply( np.median ) ).T

# Dispersion - std, max, min, range, skew, kurtosis 
d1 = pd.DataFrame( num_attributes.apply( np.std ) ).T
d2 = pd.DataFrame( num_attributes.apply( min ) ).T
d3 = pd.DataFrame( num_attributes.apply( max ) ).T
d4 = pd.DataFrame( num_attributes.apply( lambda x: x.max() - x.min() ) ).T
d5 = pd.DataFrame( num_attributes.apply( lambda x: x.skew() ) ).T
d6 = pd.DataFrame( num_attributes.apply( lambda x: x.kurtosis() ) ).T


m =pd.concat( [d2, d3, d4, ct1, ct2, d1, d5, d6] ).T.reset_index()
m.columns = ['attributes', 'min', 'max', 'range', 'mean', 'median', 'std', 'skew', 'kurtosis']

m



# + hidden=true
sns.distplot( df1['competition_distance'], kde=False )

# + [markdown] heading_collapsed=true hidden=true
# ### 1.7.2. Categories Attributes

# + hidden=true
cat_attributes.apply( lambda x: x.unique().shape[0] )

# + hidden=true
aux = df1[( df1['state_holiday'] != '0' ) & ( df1['sales'] > 0 )]

plt.subplot( 1, 3, 1 )
sns.boxplot( x='state_holiday' ,y='sales', data=aux )

plt.subplot( 1, 3, 2 )
sns.boxplot( x='store_type' ,y='sales', data=aux )

plt.subplot( 1, 3, 3 )
sns.boxplot( x='assortment' ,y='sales', data=aux )


# + [markdown] heading_collapsed=true
# # 2.0. FEATURE ENGENIREERING

# + hidden=true
df2 = df1.copy()

# + [markdown] heading_collapsed=true hidden=true
# ## 2.1. Mapa Mental de Hipóteses

# + hidden=true hide_input=false
Image('img/mind_map_hyphotesis.png')

# + [markdown] heading_collapsed=true hidden=true
# ## 2.2. Criação das Hipóteses

# + [markdown] heading_collapsed=true hidden=true
# ### 2.2.1. Hipóteses Loja

# + [markdown] hidden=true
# **1.** Lojas com número maior de funcionários deveriam vender mais.
#
# **2.** Lojas com maior capacidade de estoque deveriam vender mais.
#
# **3.** Lojas com maior porte deveriam vender mais.
#
# **4.** Lojas com maior sortimentos deveriam vender mais.
#
# **5.** Lojas com competidores mais próximos deveriam vender menos.
#
# **6.** Lojas com competidores à mais tempo deveriam vendem mais.

# + [markdown] heading_collapsed=true hidden=true
# ### 2.2.2. Hipóteses Produto

# + [markdown] hidden=true
# **1.** Lojas que investem mais em Marketing deveriam vender mais.
#
# **2.** Lojas com maior exposição de produto deveriam vender mais.
#
# **3.** Lojas com produtos com preço menor deveriam vender mais.
#
# **4.** Lojas com promoções mais agressivas ( descontos maiores ), deveriam vender mais.
#
# **5.** Lojas com promoções ativas por mais tempo deveriam vender mais.
#
# **6.** Lojas com mais dias de promoção deveriam vender mais.
#
# **7.** Lojas com mais promoções consecutivas deveriam vender mais.

# + [markdown] heading_collapsed=true hidden=true
# ### 2.2.3. Hipóteses Tempo

# + [markdown] hidden=true
# **1.** Lojas abertas durante o feriado de Natal deveriam vender mais.
#
# **2.** Lojas deveriam vender mais ao longo dos anos.
#
# **3.** Lojas deveriam vender mais no segundo semestre do ano.
#
# **4.** Lojas deveriam vender mais depois do dia 10 de cada mês.
#
# **5.** Lojas deveriam vender menos aos finais de semana.
#
# **6.** Lojas deveriam vender menos durante os feriados escolares.

# + [markdown] heading_collapsed=true hidden=true
# ## 2.3. Lista Final de Hipóteses

# + [markdown] hidden=true
# **1.** Lojas com maior sortimentos deveriam vender mais.
#
# **2.** Lojas com competidores mais próximos deveriam vender menos.
#
# **3.** Lojas com competidores à mais tempo deveriam vender mais.
#
# **4.** Lojas com promoções ativas por mais tempo deveriam vender mais.
#
# **5.** Lojas com mais dias de promoção deveriam vender mais.
#
# **6.** Lojas com mais promoções consecutivas deveriam vender mais.
#
# **7.** Lojas abertas durante o feriado de Natal deveriam vender mais.
#
# **8.** Lojas deveriam vender mais ao longo dos anos.
#
# **9.** Lojas deveriam vender mais no segundo semestre do ano.
#
# **10.** Lojas deveriam vender mais depois do dia 10 de cada mês.
#
# **11.** Lojas deveriam vender menos aos finais de semana.
#
# **12.** Lojas deveriam vender menos durante os feriados escolares.

# + [markdown] heading_collapsed=true hidden=true
# ## 2.4. Feature Engineering

# + hidden=true
# year
df2['year'] = df2['date'].dt.year

# month
df2['month'] = df2['date'].dt.month

# day
df2['day'] = df2['date'].dt.day

# week of year
df2['week_of_year'] = df2['date'].dt.weekofyear

# year week
df2['year_week'] = df2['date'].dt.strftime( '%Y-%W' )

# competition since
df2['competition_since'] = df2.apply( lambda x: datetime.datetime( year=x['competition_open_since_year'],
                                                                   month=x['competition_open_since_month'], day=1 ), axis=1 )

df2['competition_time_month'] = ( ( df2['date'] - df2['competition_since'] )/30 ).apply( lambda x: x.days ).astype( int )

# promo since
df2['promo_since'] = df2['promo2_since_year'].astype( str ) + '-' + df2['promo2_since_week'].astype( str )

df2['promo_since'] = df2['promo_since'].apply( lambda x: datetime.datetime.strptime( x + '-1', '%Y-%W-%w' ) - datetime.timedelta( days=7 ) )

df2['promo_time_week'] = ( ( df2['date'] - df2['promo_since'] )/7 ).apply( lambda x: x.days ).astype( int )

# assortment
df2['assortment'] = df2['assortment'].apply( lambda x: 'basic' if x == 'a' else 'extra' if x=='b' else 'extend' )

#  state holiday
df2['state_holiday'] = df2['state_holiday'].apply( lambda x: 'public_holiday' if x == 'a' else 
                                                             'easter_holiday' if x == 'b' else 
                                                             'christmas'      if x == 'c' else 'regular_day' )


# + [markdown] heading_collapsed=true
# # 3.0. FILTRAGEM DE VARIÁVEIS

# + hidden=true
df3 = df2.copy()

# + hidden=true
df3.head()

# + [markdown] heading_collapsed=true hidden=true
# ## 3.1. Filtragem das Linhas

# + hidden=true
df3 = df3[( df3['open'] != '0' ) & ( df3['sales'] > 0 )]

# + hidden=true
df3.head()

# + [markdown] heading_collapsed=true hidden=true
# ## 3.2. Seleção das Colunas

# + hidden=true
cols_drop = ['customers', 'open', 'promo_interval', 'month_map']
df3 = df3.drop( cols_drop, axis=1 )

# + hidden=true
df3.head()

# + [markdown] heading_collapsed=true
# # 4.0. ANALISE EXPLORATÓRIA DOS DADOS

# + hidden=true
df4 = df3.copy()

# + hidden=true
df4.head()

# + [markdown] heading_collapsed=true hidden=true
# ## 4.1. Análise Univariada

# + [markdown] heading_collapsed=true hidden=true
# ### 4.1.1. Response Variable

# + hidden=true
sns.distplot( df4['sales'], kde=False )

# + [markdown] heading_collapsed=true hidden=true
# ### 4.1.2. Numerical Variable

# + hidden=true
num_attributes = df4.select_dtypes( include= ['int64', 'float64'] )
cat_attributes = df4.select_dtypes( exclude=['int64', 'float64', 'datetime64[ns]'] )

# + hidden=true
num_attributes.hist( bins=25 );

# + [markdown] heading_collapsed=true hidden=true
# ### 4.1.3. Categorical Variable

# + hidden=true
df4['assortment'].drop_duplicates()

# + hidden=true
# state_holiday
plt.subplot( 3, 2, 1 )
a = df4[df4['state_holiday'] != 'regular_day']
sns.countplot( a['state_holiday'] )

plt.subplot( 3, 2, 2 )
sns.kdeplot( df4[df4['state_holiday'] == 'public_holiday']['sales'], label='public_holiday', shade=True )
sns.kdeplot( df4[df4['state_holiday'] == 'easter_holiday']['sales'], label='easter_holiday', shade=True )
sns.kdeplot( df4[df4['state_holiday'] == 'christmas']['sales'], label='christmas', shade=True )

# store_type
plt.subplot( 3, 2, 3 )
sns.countplot( df4['store_type'] )

plt.subplot( 3, 2, 4 )
sns.kdeplot( df4[df4['store_type'] == 'a']['sales'], label='a', shade=True )
sns.kdeplot( df4[df4['store_type'] == 'b']['sales'], label='b', shade=True )
sns.kdeplot( df4[df4['store_type'] == 'c']['sales'], label='c', shade=True )
sns.kdeplot( df4[df4['store_type'] == 'd']['sales'], label='d', shade=True )

# assortment
plt.subplot( 3, 2, 5 )
sns.countplot( df4['store_type'] )

plt.subplot( 3, 2, 6 )
sns.kdeplot( df4[df4['assortment'] == 'basic']['sales'], label='basic', shade=True )
sns.kdeplot( df4[df4['assortment'] == 'extra']['sales'], label='extra', shade=True )
sns.kdeplot( df4[df4['assortment'] == 'extend']['sales'], label='extend', shade=True )


# + [markdown] heading_collapsed=true hidden=true
# ## 4.2. Análise Bivariada

# + hidden=true
df4.head()

# + [markdown] heading_collapsed=true hidden=true
# ### **H1.** Lojas com maior sortimentos deveriam vender mais.
# **FALSA** Lojas com COMPETIDORES MAIS PROXIMOS vendem MAIS 

# + hidden=true hide_input=false
aux1 = df4[['assortment', 'sales']].groupby( 'assortment' ).sum().reset_index()
sns.barplot( x='assortment', y='sales', data=aux1 )

aux2 = df4[['year_week', 'assortment', 'sales']].groupby( ['year_week', 'assortment'] ).sum().reset_index()
aux2.pivot( index='year_week', columns='assortment', values='sales' ).plot()

aux3 = aux2[aux2['assortment'] == 'extra']
aux3.pivot( index='year_week', columns='assortment', values='sales' ).plot()


# + [markdown] heading_collapsed=true hidden=true
# ### **H2.** Lojas com competidores mais próximos deveriam vender menos.
# **FALSA** Lojas com COMPETIDORES MAIS PROXMIMOS vendem MAIS.

# + hidden=true
aux1 = df4[['competition_distance', 'sales']].groupby( 'competition_distance' ).sum().reset_index()

plt.subplot( 1, 3, 1 )
sns.scatterplot( x='competition_distance', y='sales', data=aux1 );

plt.subplot( 1, 3, 2 )
bins = list( np.arange(0, 20000, 1000) ) 
aux1['competition_distance_binned'] = pd.cut( aux1['competition_distance'], bins=bins )
aux2 = aux1[['competition_distance_binned', 'sales']].groupby( 'competition_distance_binned' ).sum().reset_index()
sns.barplot( x='competition_distance_binned', y='sales', data=aux2 );

plt.subplot( 1, 3, 3 )
sns.heatmap( aux1.corr( method='pearson' ), annot=True )


# + [markdown] heading_collapsed=true hidden=true
# ### **H3.** Lojas com competidores à mais tempo deveriam vender mais.
# **FALSA** Lojas com COMPETIDORES À MAIS TEMPO vendem MENOS.

# + hidden=true
plt.subplot( 1, 3, 1 )
aux1 = df4[['competition_time_month', 'sales']].groupby( 'competition_time_month' ).sum().reset_index()
aux2 = aux1[( aux1['competition_time_month'] < 120 ) & ( aux1['competition_time_month'] != 0 )]
sns.barplot( x='competition_time_month', y='sales', data=aux2 );
plt.xticks( rotation=90 );

plt.subplot( 1, 3, 2 )
sns.regplot( x='competition_time_month', y='sales', data=aux2 );

plt.subplot( 1, 3, 3 )
sns.heatmap( aux1.corr( method='pearson' ), annot=True )


# + [markdown] heading_collapsed=true hidden=true
# ### **H4.** Lojas com promoções ativas por mais tempo deveriam vender mais.
# **FALSA** Loja com PROMOÇÔES ATIVAS POR MAIS TEMPO vendem MENOS, depois de um certo tempo de promoção.

# + hidden=true
aux1 = df4[['promo_time_week', 'sales']].groupby( 'promo_time_week' ).sum().reset_index()

grid = GridSpec( 2,3 )

plt.subplot( grid[0,0] )
aux2 = aux1[aux1['promo_time_week'] > 0] # promo extendido
sns.barplot( x='promo_time_week', y='sales', data=aux2 );
plt.xticks( rotation=90 );

plt.subplot( grid[0,1] )
sns.regplot( x='promo_time_week', y='sales', data=aux2 );

plt.subplot( grid[1,0] )
aux3 = aux1[aux1['promo_time_week'] < 0] # promo regular
sns.barplot( x='promo_time_week', y='sales', data=aux3 );
plt.xticks( rotation=90 );

plt.subplot( grid[1,1] )
sns.regplot( x='promo_time_week', y='sales', data=aux3 );

plt.subplot( grid[:,2] )
sns.heatmap( aux1.corr( method='pearson' ), annot=True );



# + [markdown] heading_collapsed=true hidden=true
# ### **H5.** <s> Lojas com mais dias de promoção deveriam vender mais. </s>

# + [markdown] heading_collapsed=true hidden=true
# ### **H6.** Lojas com mais promoções consecutivas deveriam vender mais.
# **FALSA** Lojas com mais promoções consecutivas vendem menos

# + hidden=true
df4.columns

# + hidden=true
df4[['promo', 'promo2', 'sales']].groupby( ['promo', 'promo2'] ).sum().reset_index()

# + hidden=true
aux1 = df4[( df4['promo'] == 1 ) & ( df4['promo2'] == 1 )][['year_week','sales']].groupby( 'year_week' ).sum().reset_index() 
ax = aux1.plot()

aux2 = df4[( df4['promo'] == 1 ) & ( df4['promo2'] == 0 )][['year_week','sales']].groupby( 'year_week' ).sum().reset_index()
aux2.plot( ax=ax )

ax.legend( labels=['Tradicional & Extendida', 'Tradicional']);

# + [markdown] heading_collapsed=true hidden=true
# ### **H7.** Lojas abertas durante o feriado de Natal deveriam vender mais.
# **FALSA** Lojas abertas durante o feriado de Natal vendem menos.

# + hidden=true
aux = df4[df4['state_holiday'] != 'regular_day']

plt.subplot( 1, 2, 1 )
aux1 = aux[['state_holiday', 'sales']].groupby( 'state_holiday' ).sum().reset_index()
sns.barplot( x='state_holiday', y='sales', data=aux1 );

plt.subplot( 1, 2, 2 )
aux2 = aux[['year', 'state_holiday', 'sales']].groupby( ['year', 'state_holiday'] ).sum().reset_index()
sns.barplot( x='year', y='sales', hue='state_holiday', data=aux2 );


# + [markdown] heading_collapsed=true hidden=true
# ### **8.** Lojas deveriam vender mais ao longo dos anos.
# **FALSA** Lojas vendem menos ao longo dos anos.

# + hidden=true
aux1 = df4[['year', 'sales']].groupby( 'year' ).sum().reset_index()

plt.subplot( 1, 3, 1 )
sns.barplot( x='year', y='sales', data=aux1 );

plt.subplot( 1, 3, 2 )
sns.regplot( x='year', y='sales', data=aux1 );

plt.subplot( 1, 3, 3 )
sns.heatmap( aux1.corr( method='pearson' ), annot=True )

# + [markdown] heading_collapsed=true hidden=true
# ### **9.** Lojas deveriam vender mais no segundo semestre do ano.
# **FALSA** Lojas vendem menos no segundo semestre

# + hidden=true
aux1 = df4[['month', 'sales']].groupby( 'month' ).sum().reset_index()

plt.subplot( 1, 3, 1 )
sns.barplot( x='month', y='sales', data=aux1 );

plt.subplot( 1, 3, 2 )
sns.regplot( x='month', y='sales', data=aux1 );

plt.subplot( 1, 3, 3 )
sns.heatmap( aux1.corr( method='pearson' ), annot=True )

# + [markdown] heading_collapsed=true hidden=true
# ### **10.** Lojas deveriam vender mais depois do dia 10 de cada mês.
# **VERDADEIRA** Lojas vendem mais depois do dia 10 de cada mês

# + hidden=true
aux1 = df4[['day', 'sales']].groupby( 'day' ).sum().reset_index()

plt.subplot( 2, 2, 1 )
sns.barplot( x='day', y='sales', data=aux1 );

plt.subplot( 2, 2, 2 )
sns.regplot( x='day', y='sales', data=aux1 );

plt.subplot( 2, 2, 3 )
sns.heatmap( aux1.corr( method='pearson' ), annot=True );

aux1['before_after'] = aux1['day'].apply( lambda x: 'before_10_days' if x <=10 else 'after_10_days' )
aux2 = aux1[['before_after', 'sales']].groupby( 'before_after' ).sum().reset_index()

plt.subplot( 2, 2, 4 )
sns.barplot( x='before_after', y='sales', data=aux2 );



# + [markdown] heading_collapsed=true hidden=true
# ### **11.** Lojas deveriam vender menos aos finais de semana.
# **VERDADEIRA** Lojas vendem menos aos finais de semana

# + hidden=true
aux1 = df4[['day_of_week', 'sales']].groupby( 'day_of_week' ).sum().reset_index()

plt.subplot( 1, 3, 1 )
sns.barplot( x='day_of_week', y='sales', data=aux1 );

plt.subplot( 1, 3, 2 )
sns.regplot( x='day_of_week', y='sales', data=aux1 );

plt.subplot( 1, 3, 3 )
sns.heatmap( aux1.corr( method='pearson'), annot=True );


# + [markdown] heading_collapsed=true hidden=true
# ### **12.** Lojas deveriam vender menos durante os feriados escolares.
# **VERDADEIRA** Lojas vendem menos durante os feriados escolares, exceto julho e agosto

# + hidden=true
aux1 = df4[['school_holiday', 'sales']].groupby( 'school_holiday' ).sum().reset_index()

plt.subplot( 2, 1, 1 )
sns.barplot( x='school_holiday', y='sales', data=aux1 );
plt.subplot( 2, 1, 2 )
aux2 = df4[['month', 'school_holiday', 'sales']].groupby( ['month', 'school_holiday'] ).sum().reset_index()
sns.barplot( x='month', y='sales', hue='school_holiday', data=aux2 );

# + [markdown] hidden=true
# ### 4.2.1. Resumo das Hipóteses

# + hidden=true
from tabulate import tabulate

# + hidden=true
tab = [['Hipoteses', 'Conclusao', 'Relevancia'], 
 ['H1', 'Falsa', 'Baixa'],
 ['H2', 'Falsa', 'Media'],
 ['H3', 'Falsa', 'Media'],
 ['H4', 'Falsa', 'Baixa'],
 ['H5', '-', '-'],
 ['H6', 'Falsa', 'Baixa'],
 ['H7', 'Falsa', 'Media'],
 ['H8', 'Falsa', 'Alta'],
 ['H9', 'Falsa', 'Alta'],
 ['H10', 'Verdadeira', 'Alta'],
 ['H11', 'Verdadeira', 'Alta'],
 ['H12', 'Verdadeira', 'Baixa']]

print( tabulate( tab, headers='firstrow') )

# + hidden=true
## 4.3. Análise Multivariada

# + hidden=true
### 4.3.1. Numerical Attributes

# + hidden=true
correlation = num_attributes.corr( method='pearson' )
sns.heatmap( correlation, annot=True );

# + [markdown] hidden=true
# ### 4.3.2. Category Attributes 

# + hidden=true
# only categorical data
a = df4.select_dtypes( include='object' )

# calculate Cramer V
a1 = cramer_v( a['state_holiday'], a['state_holiday'] )
a2 = cramer_v( a['state_holiday'], a['store_type'] )
a3 = cramer_v( a['state_holiday'], a['assortment'] )

a4 = cramer_v( a['store_type'], a['state_holiday'] )
a5 = cramer_v( a['store_type'], a['store_type'] )
a6 = cramer_v( a['store_type'], a['assortment'] )

a7 = cramer_v( a['assortment'], a['state_holiday'] )
a8 = cramer_v( a['assortment'], a['store_type'] )
a9 = cramer_v( a['assortment'], a['assortment'] )

# final dataset

d =pd.DataFrame( { 'state_holiday': [a1, a2, a3], 
                   'store_type':[a2, a3, a4], 
                   'assortment':[a7, a8, a9] } )

d = d.set_index( d.columns )

sns.heatmap( d, annot=True )

# + [markdown] heading_collapsed=true
# # 5.0. DATA PREPARATION

# + hidden=true
df5 = df4.copy()

# + [markdown] heading_collapsed=true hidden=true
# ## 5.1. Normalization

# + hidden=true


# + [markdown] heading_collapsed=true hidden=true
# ## 5.2. Rescaling

# + hidden=true
rs = RobustScaler()
mms = MinMaxScaler()

# competition distance
df5['competition_distance'] = rs.fit_transform( df5[['competition_distance']].values )

# competition time month
df5['competition_time_month'] = rs.fit_transform( df5[['competition_time_month']].values )

# promo time week
df5['promo_time_week'] = mms.fit_transform( df5[['promo_time_week']].values)

# year
df5['year'] = mms.fit_transform( df5[['year']].values)

# + [markdown] heading_collapsed=true hidden=true
# ## 5.3. Transfromation

# + [markdown] hidden=true
# ### 5.3.1. Encoding

# + hidden=true
# state_holiday - One Hot Encoding
df5 = pd.get_dummies( df5, prefix=['state_holiday'], columns=['state_holiday'])

# store_type - Label Encoding
le = LabelEncoder()
df5['assortment'] = le.fit_transform( df5['assortment'] )

# assortment - Ordinal Encoding
assortment_dict = {'basic': 1, 'extra': 2, 'extended': 3}
df5['assortment'] = df5['assortment'].map( assortment_dict )


# + [markdown] hidden=true
# ### 5.3.2. Respose Variable Transformation

# + hidden=true
df5['sales'] = np.log1p( df5['sales'] )

# + [markdown] hidden=true
# ### 5.3.3. Nature Transformation

# + hidden=true
# day of week 
df5['day_of_week_sin'] = df5['day_of_week'].apply( lambda x: np.sin( x * ( 2. * np.pi/7 ) ) )
df5['day_of_week_cos'] = df5['day_of_week'].apply( lambda x: np.cos( x * ( 2. * np.pi/7 ) ) )

# month
df5['month_sin'] = df5['month'].apply( lambda x: np.sin( x * ( 2. * np.pi/12 ) ) )
df5['month_cos'] = df5['month'].apply( lambda x: np.cos( x * ( 2. * np.pi/12 ) ) )

# day
df5['day_sin'] = df5['day'].apply( lambda x: np.sin( x * ( 2. * np.pi/30 ) ) )
df5['day_cos'] = df5['day'].apply( lambda x: np.cos( x * ( 2. * np.pi/30 ) ) )

# week of year
df5['week_of_year_sin'] = df5['week_of_year'].apply( lambda x: np.sin( x * ( 2. * np.pi/52 ) ) )
df5['week_of_year_cos'] = df5['week_of_year'].apply( lambda x: np.cos( x * ( 2. * np.pi/52 ) ) )

# + hidden=true
df5.head()

# + hidden=true


StarcoderdataJupyterScriptsDedupFiltered