# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# # Machine Learning Engineer Nanodegree
# ## Capstone Project - InstaCart Market Basket Analysis
#
# ### Utility functions for pre-processing and feature engineering
# This notebook duplicates the functions I created for pre-processing and building new features for the project. 
# I have included them as a separate notebook for easier viewing and better way to document the code
#
# The project uses pandas v0.18 and relies on saving dataframes in hdfs format (pandas native) for efficiency. This option requires the `tables` (pytables) module to be installed on the host machine. I needed to install this in my Anaconda2 environment using `conda`.
#
# The first block here is boilerplate code, used by other notebooks I created for downstream use. Importing gc helps with managing the memory footprint during the pre-processing and analysis stages.

# + active=""
# import numpy as np
# import pandas as pd
# import os
# import time
# import gc
#
# print "Loaded numpy and pandas libraries"
#
# # The _datapath variable should point to the folder where the hdfs files are located
# _datapath = './'
# os.chdir(_datapath)
#
# # Named hdfs stores
# RAW_STORE = 'instacart_raw.hdf5' # direct copies of the csv files in hdfs format
# HDF_STORE = 'instacart.hdf5'     # stores the augmented products_priors dataframe which is used to produce the eng. features
# FVARS_STORE = 'features.hdf5'    # stores engineered features for use in analysis
# RANDOM_STATE = 46
#
# def get_from_hdf(tnames, STORE=RAW_STORE):
#     hnames = {}
#     result = {}
#     for n in tnames:
#         name = hname[n] if n in hnames else n
#         try:
#             print "Loading {} datasets ...".format(n)
#             result[n] = pd.read_hdf(STORE, name)
#         except Exception as e:
#             print e
#             print "Dataset could not be loaded. Is the hdf store missing?"
#     return result
# -

# ### Augmenting orders and products
#
# - `get_sample` : is a utility function to enable us to take a small randomized sample of a large dataframe during development
# - `preprocess_orders` : handles missing values and outliers in the `orders` table. The `days_since_prior_order` uses a value of _30_ for true values greater than or equal to 30. We modified this to a random number between 30 and 50 and added a categorical variable to indicate that those values are assigned during preprocessing. _Additionally_, this routine also calculates the cumulative number of days between a user's orders.
# - `add_product_groups` : adds `department_id` and `aisle_id` columns to the order history (`priors`)
# - `get_hist_chunk` : another function to assist development and code testing. This takes a random sample (size determined by `frac`) of users and then selects their order history. This enables us to go through the pre-processing logic in small chunks to verify that the logic is working correctly.

# + active=""
# def get_sample(p, **kw):
#     r"""
#         requires two keyword inputs 
#         by = column name
#         frac = fraction of rows to sample
#     """    
#     try:
#         colname = kw['by']
#         frac = kw['frac'] if frac in kw else 0.05
#         u = p[[colname]].unique()
#         u = u.sample(frac=frac, random_state=seed)
#         ochunk = p[p[[colname]].isin(u)]
#     except NameError as e:
#         ochunk = ""
#         print e
#     finally:
#         del u
#     print "Selected {}={:.2f}% rows of {} {}.".format(len(ochunk),\
#             100.0*frac, int(len(u)*frac), colname)
#     return ochunk
#
# def preprocess_orders(orders):
#
#     orders.sort_values(by=['user_id', 'order_number'], inplace=True)
#
#     # Add additional columns
#     x = orders.days_since_prior_order
#     orders['30plus_days'] = (x==30).astype('int8')
#
#     # Impute 'missing' days_since_prior_order values
#     orders.days_since_prior_order.fillna(orders.days_since_prior_order.mean(), inplace=True)
#     ser=orders[x==30].index
#     orders.days_since_prior_order[ser] = pd.DataFrame(np.random.randint(30,50,len(ser)), index=ser)
#     orders['csum_ds'] = orders.groupby('user_id')['days_since_prior_order'].transform('cumsum') 
#
#     # Transform
#     orders['log_ds'] = np.log(orders.days_since_prior_order+1)
#
# def add_product_groups(p, products):
#     return p.merge(products[['product_id', 'department_id', 'aisle_id']], on='product_id')
#
# def get_hist_chunk(orders, priors, frac=0.05, seed=RANDOM_STATE, **kw):
#     if 'start' in kw:
#         start = kw['start']
#         chunk = kw['size']
#         ochunk = orders.iloc[start:start+chunk]
#         print "Selected {} orders {}:{}.".format(len(ochunk), start, start+chunk)
#     elif 'all' in kw:
#         ochunk = orders
#         print "Selected all orders."
#     else:
#         u = orders.groupby('user_id')['order_id'].last().reset_index()[['user_id']]
#         u = u.sample(frac=frac, random_state=seed)
#         ochunk = u.merge(orders, how="inner", on='user_id')
#         print "Selected {} orders for {} users.".format(len(ochunk), len(u))
#     ochunk.sort_values(by=['user_id', 'order_number'], inplace=True)
#     p = ochunk.merge(priors, on='order_id')
#     return p
# -

# There is one feature that required special handling. We are given the days between orders, but a product may not be on each order placed by a user. In order to calculate the average number of days between when a user orders a specific product, we took these steps:
#  1. sort the data by `user_id`, `product_id`, and `order_number`
#  2. take the difference of the cumulative sum of days since the last order (calculated in preprocess_orders and stored as `csum_ds`) of each row and its previous one. For the first row, we use the days since the previous order
#  
#  While there is a way to calculate this in `pandas` using a lambda function, the computation took a very long time. In the interest of efficiency, I wrote a function (`diff1`) to achieve this using `numpy` matrices. TI was able to achive a 20-fold speedup using this method (estimate based on small samples).

# + active=""
# def diff_1(m):
#     r""" Takes a matrix where each row is strucured as [user_id, product_id, order_number and cumulative sum of
#         the number of days since previous order. 
#     """
#     user = product = ""
#     last = 0
#     for row in m:  
#         if (user == row[0]) and (product == row[1]): 
#             # another order for the same user and product
#             save = row[3]  
#             row[3] -= last # diff between this row's cumsum and the previous row for this product and user
#             last = save
#         else:
#             # got a different use or product, so set the 'current' user and product
#             user = row[0]
#             product = row[1]
#             last = row[3] # set the last value for cumsum of this user/product combo
#
#     return m
#
# def user_product_ds_last(p):
#
#         p2 = p[['user_id', 'product_id', 'order_number', 'csum_ds']]\
#                 .sort_values(by=['user_id', 'product_id', 'order_number'])
#         g = pd.DataFrame(diff_1(p2.as_matrix()), dtype='int32')
#         g.columns = ['user_id', 'product_id', 'order_number', 'ds_last']
#         print len(g), g.columns 
#         return g
# -

# ### Building user-product features
#
# Building features for user-product combinations using extensive use of Dataframe `aggregate` function. Descriptions of these features are provided in the Preprocessing notebook

# + active=""
# def user_product_features(phist, verbosity=3):
#
#     tic = tic0 = time.time()
#     try: # inline comments
#         up_feat = phist.groupby(['user_id', 'aisle_id', 'product_id'])\
#                 .agg({'reordered':['count', 'mean', 'last'], 'ds_last':'mean',\
#                     'order_number':'max', 'add_to_cart_order':'mean'})\
#                 .reset_index()
#         up_feat.columns=['user_id', 'aisle_id', 'product_id', 'up_cart_rank', 'up_avg_days',\
#              'up_in_last', 'up_times', 'up_reord_prob', 'up_last_reord']
#         up_feat['up_in_last'] = up_feat['up_in_last'].astype('int8')
#         toc = time.time()
#         if verbosity > 2:
#             print "Step 1 / 4 - computed user-product averages {:.3f} s".format(toc - tic)
#             tic = toc
#
#         u_orders = phist.groupby(['user_id', 'order_id'])['product_id'].count().reset_index()
#         u_orders = u_orders.groupby('user_id').agg({'order_id':'nunique', 'product_id':'mean'})\
#                         .reset_index()
#         u_orders.rename(columns={'order_id':'uo_count', 'product_id':'ubasket_avg'},\
#                     inplace=True)
#         up_feat = up_feat.merge(u_orders, on='user_id')
#         up_feat["up_prob"] = up_feat.up_times / up_feat.uo_count
#         del u_orders
#         toc = time.time()
#         if verbosity > 2:
#             print "Step 2 / 4 - computed averages by order {:.3f} s".format(toc-tic)
#             tic = toc
#
#         up_orders = phist.groupby(['user_id', 'product_id'])\
#                     ['order_dow', 'order_hour_of_day', '30plus_days' ].mean()\
#                     .reset_index()
#         up_orders.rename(columns={'order_dow':'up_dow', 'order_hour_of_day':'up_tod',\
#                      '30plus_days':'up_30_avg'}, inplace=True)
#         up_feat = up_feat.merge(up_orders, on=['user_id', 'product_id'])
#         del up_orders
#
#         toc = time.time()
#         if verbosity > 2:
#             print "Step 3 / 4 - computed average by product {:.3f} s".format(toc-tic)
#             tic = toc
#
#         pa_up = phist.groupby(['user_id', 'order_id', 'aisle_id'])\
#             .agg({'product_id':'nunique',\
#                 'reordered':'sum'}).reset_index()
#         pa_up.columns = ['user_id', 'order_id', 'aisle_id', 'u_a_pcount', 'u_a_reord']
#         pa_up = pa_up.groupby(['user_id', 'aisle_id'])\
#                 .agg({'order_id':'count', 'u_a_pcount':'mean', 'u_a_reord':'mean'})\
#                 .reset_index()
#         pa_up.rename(columns={'order_id':'ua_ocount'}, inplace=True)
#         print pa_up.columns
#         up_feat = up_feat.merge(pa_up, on=['user_id', 'aisle_id'])
#         toc = time.time()
#         if verbosity > 2:
#             print "Step 4 / 4 {:.3f} s".format(toc - tic)
#             tic = toc
#     except:
#         up_feat = None
#
#     toc = time.time()
#     if verbosity > -1:
#         print "Completed in a total of {:.3f} s".format(toc - tic0)
#         tic = toc
#         print up_feat.columns
#     return up_feat
# -

# ### Building user's top-_N_ aisles feature
#
# The function below analyses users' puchase patterns across orders to identify which groups of items they tend to purchase most frequently. In this round, we are using aisles as a convenient group for products. The function counts the number of products per aisle, sorts them in descending order, then picks the aisle-ids for the top-N (N is an input) aisles. 
#
# I rely on some matrix manipulation to convert the top-N list to a data frame with N-columns. 

# + active=""
# def build_user_topn_aisles(p, topN=10):
#
#     r"""
#         Create dataframe with the top-N aisles a user shops in
#         Input is a df from the priors table, with user_id and aisle_id
#     """
#
#     print "{} products ordered".format(len(p))
#     gg = p.groupby(['user_id', 'order_id', 'aisle_id'])['product_id'].count().reset_index()
#     print " ..  including products from {} aisles".format(len(gg))
#     gg = gg.groupby(['user_id', 'aisle_id'])['product_id'].sum().reset_index()\
#             .rename(columns={'product_id':'oa_count'})
#     gg = gg.sort_values(by=['user_id', 'oa_count'], ascending=False)[['user_id', 'aisle_id']]
#     gm = gg.groupby('user_id')['aisle_id'].unique().reset_index().as_matrix()
#     print " .. {} users".format(gm.shape[0])
#
#     dm = np.zeros((topN+1)*len(gm)).reshape(len(gm), topN+1)
#     for i in xrange(len(gm)):
#         dm[i] = np.concatenate([[gm[i][0]], gm[i][1], np.zeros(topN)])[:topN+1]
#     g = pd.DataFrame(dm, dtype="int32")
#     g.columns=['user_id']+['top-{}'.format(x) for x in range(1,topN+1)]
#
#     return g
# -

# ### Benchmark Models and prediction functions

# + active=""
# # ---------- Benchmark 1 -- 
# def bench1_fit(priors, orders):
#     pset = orders[orders.eval_set==0][['order_id', 'user_id']]
#     bench = priors.merge(pset,  on='order_id')
#
#     # Calculate product popularity
#     popular = bench[bench.reordered==1].groupby('product_id')['order_id'].count().reset_index()\
#     .rename(columns={'order_id':'frequency'})
#     popular.sort_values(by='frequency', ascending=False, inplace=True)
#
#     #Calculate average basket size
#     basket_size = bench.groupby('order_id')['product_id'].count()\
#     .reset_index().rename(columns={'product_id':'Average Size'})
#     topN = int(round(np.float(basket_size[[1]].mean()[0]), 0))
#     print "Average basket size is {:.3f}".format(topN)
#     
#     model = popular.product_id[:topN].reset_index().drop('index', axis=1)
#     model['in_order'] = (np.ones(len(model))).astype('uint8')
#     return model
#
# # ---------- Benchmark 2 -- 
# def bench2_fit(priors, orders):
#     # Add user_id to the historical set
#     pset = orders[orders.eval_set==0][['order_id', 'user_id', 'order_number']]
#     bench = priors.merge(pset,  on='order_id')
#     model = bench.groupby(['user_id', 'product_id'])['order_number'].last().reset_index()
#     model.rename(columns={'last':'in_order', 'order_number':'in_order'}, inplace=True)
#     model.in_order = (model.in_order / model.in_order).astype('uint8')
#
#     return model
#
# # ---------- Benchmark 3 -- 
# def bench3_fit(priors, orders):
#     # Find the history set -- prior and train
#     bench = orders[orders.eval_set==0][['order_id', 'user_id', 'order_number']]
#     bench3 = bench.merge(priors, on='order_id')
#     # Calculate most frequently purchased items by user
#     popular = bench3.groupby(['user_id','product_id'])['order_id'].count().reset_index()\
#                 .rename(columns={'order_id':'frequency'})
#     popular['prod_rank']=popular.groupby(['user_id'])['frequency']\
#                 .rank(ascending=False).astype('uint16')
#
#     #Calculate average basket size
#     u_baskets = bench3.groupby(['user_id', 'order_id'])['product_id'].count()\
#         .reset_index().rename(columns={'product_id':'basket_size'})
#     u_baskets = u_baskets.groupby('user_id')['basket_size'].mean().reset_index()
#     model = popular.merge(u_baskets, on='user_id') 
#     model['in_order'] = model.prod_rank[model.prod_rank<model.basket_size]
#     return model[['user_id', 'product_id', 'in_order']]
#
# # ---------- Benchmark predictions -- 
#
# def	bench_predict(model, X):
#     # Input X should be a dataframe including a user_id
#     # Input model should be a dataframe with two or three columns:
#     #		[user_id], product_id and in_order
#
#     if 'user_id' in model.columns:
#         X = X.merge(model[['user_id','product_id','in_order']], how='left', 
#             on=['user_id', 'product_id'])
#     else:
#         X = X.merge(model, how='left', on='product_id')
#     pred = X.in_order.fillna(0)
#
#     return pred
# -



StarcoderdataJupyterScriptsDedupFiltered