# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # SETTINGS
#
# This notebook performs initial data processing:
# - importing the raw data
# - converting feature types
# - merging some data.frames
# - saving data as two CSV files: `orders.csv` and `items.csv`.
#
# A detailed walkthrough of the code covering the key steps is provided in [this blog post](https://kozodoi.me/python/time%20series/demand%20forecasting/competitions/2020/07/27/demand-forecasting.html).

# +
##### LIBRARIES

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns

import scipy.stats

import os
import time
import datetime
import random
import multiprocessing
import pickle
import warnings
import gc
import sys

# +
##### MODULES

sys.path.append('../codes')  

from data_prep import print_factor_levels, print_missings, find_constant_features, split_nested_features
from versioning import save_csv_version

# +
##### SETTINGS

warnings.filterwarnings('ignore')
pd.set_option('display.max_columns', None)
plt.style.use('dark_background')
# %matplotlib inline
gc.enable()
# -

# # DATA IMPORT

# +
##### IMPORT

infos  = pd.read_csv('../data/raw/infos.csv',  sep = '|')
items  = pd.read_csv('../data/raw/items.csv',  sep = '|')
orders = pd.read_csv('../data/raw/orders.csv', sep = '|')

print(infos.shape)
print(items.shape)
print(orders.shape)
# -

infos.head()

items.head()

orders.head()

# # PROCESSING

# ### MERGE INFOS AND ITEMS

# +
##### MERGER

print(infos.shape)
print(items.shape)
items = pd.merge(infos, items, on = 'itemID', how = 'left')
print(items.shape)
del infos
# -

# ### CONVERT FEATURE TYPES

print('-' * 50)
print(items.dtypes)
print('-' * 50)
print(orders.dtypes)
print('-' * 50)

# +
# items
for var in ['itemID', 'brand', 'manufacturer', 'category1', 'category2', 'category3']:
    items[var] = items[var].astype('str').astype('object') 
    
# orders
for var in ['transactID', 'itemID']:
    orders[var] = orders[var].astype('str').astype('object') 
    
# dates
orders['time'] = pd.to_datetime(orders['time'].astype('str'), infer_datetime_format = True)
# -

# ### CHECK FEATURES

print_factor_levels(items, top = 3)

print_factor_levels(orders, top = 3)

find_constant_features(items)

find_constant_features(orders)

# ### MISSING VALUES

# change zeros to NA where relvant
items.loc[items['brand']          == '0', 'brand']        = np.nan
items.loc[items['customerRating'] == 0, 'customerRating'] = np.nan

print_missings(items)

print_missings(orders)

# ### UNFOLD PROMOTIONS

# split promotion feature
items = split_nested_features(items, split_vars = 'promotion', sep = ',')
items.head()

# +
# convert date types
promotion_vars = items.filter(like = 'promotion_').columns
for var in promotion_vars:
    items[var] = pd.to_datetime(items[var], infer_datetime_format = True)
    
items.dtypes
# -

# # EXPORT

# save data frame
# save_csv_version() automatically adds version number to prevent overwriting
save_csv_version('../data/prepared/orders.csv', orders, index = False, compression = 'gzip')
save_csv_version('../data/prepared/items.csv',  items,  index = False, compression = 'gzip')
print(orders.shape)
print(items.shape)

StarcoderdataJupyterScriptsDedupFiltered