# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] _cell_guid="b1076dfc-b9ad-4769-8c92-a6c4dae69d19" _uuid="8f2839f25d086af736a60e9eeb907d3b93b6e0e5" papermill={"duration": 0.022639, "end_time": "2022-02-20T01:16:21.506238", "exception": false, "start_time": "2022-02-20T01:16:21.483599", "status": "completed"} tags=[]
# # H&M Recommendation: Popularity Baseline
#
# #### What we'll be doing here:
# This is a bare minimum notebook. As we have already found out, popularity and repetition is king in this competition. We'll combine these two to create a good enough baseline.

# + papermill={"duration": 0.035567, "end_time": "2022-02-20T01:16:21.564781", "exception": false, "start_time": "2022-02-20T01:16:21.529214", "status": "completed"} tags=[]
import numpy as np
import pandas as pd
import os
import glob
#import reco
from tqdm import tqdm
import datetime

# + [markdown] papermill={"duration": 0.021297, "end_time": "2022-02-20T01:16:21.607869", "exception": false, "start_time": "2022-02-20T01:16:21.586572", "status": "completed"} tags=[]
# ### Forming Train Set

# + papermill={"duration": 70.720568, "end_time": "2022-02-20T01:17:32.349928", "exception": false, "start_time": "2022-02-20T01:16:21.629360", "status": "completed"} tags=[]
data = pd.read_csv("../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv", dtype={'article_id':str})
data.head()

# + [markdown] papermill={"duration": 0.022023, "end_time": "2022-02-20T01:17:32.394392", "exception": false, "start_time": "2022-02-20T01:17:32.372369", "status": "completed"} tags=[]
# We'll drop everything except the last few(up for experimentation) days. The info from previous months are not coming of much use. 
# We'll keep 2 weeks as train and the last week as validation.

# + papermill={"duration": 16.166712, "end_time": "2022-02-20T01:17:48.583309", "exception": false, "start_time": "2022-02-20T01:17:32.416597", "status": "completed"} tags=[]
print("All Transactions Date Range: {} to {}".format(data['t_dat'].min(), data['t_dat'].max()))

data["t_dat"] = pd.to_datetime(data["t_dat"])
data = data.loc[data["t_dat"] >= datetime.datetime(2020,9,1)]

split_date = datetime.datetime(2020,9,15)
val = data.loc[data['t_dat'] > split_date, :]
train = data.loc[data['t_dat'] <= split_date, :]

# + [markdown] papermill={"duration": 0.022364, "end_time": "2022-02-20T01:17:48.628414", "exception": false, "start_time": "2022-02-20T01:17:48.606050", "status": "completed"} tags=[]
# Items which an user has bought in our train set time.

# + papermill={"duration": 3.580084, "end_time": "2022-02-20T01:17:52.231224", "exception": false, "start_time": "2022-02-20T01:17:48.651140", "status": "completed"} tags=[]
# List of all purchases per user (has repetitions)
positive_items_per_user = train.groupby(['customer_id'])['article_id'].apply(list)
positive_items_per_user

# + [markdown] papermill={"duration": 0.024545, "end_time": "2022-02-20T01:17:52.279902", "exception": false, "start_time": "2022-02-20T01:17:52.255357", "status": "completed"} tags=[]
# Next we do exponential weighting based popularity for items. This leads to items bought more recently having more weight in the popularity list.

# + papermill={"duration": 12.736656, "end_time": "2022-02-20T01:18:05.039425", "exception": false, "start_time": "2022-02-20T01:17:52.302769", "status": "completed"} tags=[]
train['pop_factor'] = train['t_dat'].apply(lambda x: 1/np.exp((datetime.datetime(2020,9,16) - x).days))
popular_items_group = train.groupby(['article_id'])['pop_factor'].sum()
_, popular_items = zip(*sorted(zip(popular_items_group, popular_items_group.keys()))[::-1])

train['pop_factor'].describe()


# + [markdown] papermill={"duration": 0.024327, "end_time": "2022-02-20T01:18:05.090983", "exception": false, "start_time": "2022-02-20T01:18:05.066656", "status": "completed"} tags=[]
# ### Moving on to Validation ...

# + papermill={"duration": 0.035593, "end_time": "2022-02-20T01:18:05.150968", "exception": false, "start_time": "2022-02-20T01:18:05.115375", "status": "completed"} tags=[]
def apk(actual, predicted, k=12):
    if len(predicted)>k:
        predicted = predicted[:k]

    score = 0.0
    num_hits = 0.0

    for i,p in enumerate(predicted):
        if p in actual and p not in predicted[:i]:
            num_hits += 1.0
            score += num_hits / (i+1.0)

    if not actual:
        return 0.0

    return score / min(len(actual), k)

def mapk(actual, predicted, k=12):
    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])


# + [markdown] papermill={"duration": 0.023848, "end_time": "2022-02-20T01:18:05.199606", "exception": false, "start_time": "2022-02-20T01:18:05.175758", "status": "completed"} tags=[]
# Items bought by users in the validation period. Similar as the one for train set.

# + papermill={"duration": 1.654206, "end_time": "2022-02-20T01:18:06.877821", "exception": false, "start_time": "2022-02-20T01:18:05.223615", "status": "completed"} tags=[]
positive_items_val = val.groupby(['customer_id'])['article_id'].apply(list)
positive_items_val, len(positive_items_val)

# + papermill={"duration": 0.504856, "end_time": "2022-02-20T01:18:07.409093", "exception": false, "start_time": "2022-02-20T01:18:06.904237", "status": "completed"} tags=[]
# creating validation set for metrics use case
val_users = positive_items_val.keys()
val_items = []

for i,user in tqdm(enumerate(val_users)):
    val_items.append(positive_items_val[user])
    
print("Total users in validation:", len(val_users))

# + [markdown] papermill={"duration": 0.027505, "end_time": "2022-02-20T01:18:07.465171", "exception": false, "start_time": "2022-02-20T01:18:07.437666", "status": "completed"} tags=[]
# We'll now validate our algo on the validation set.

# + papermill={"duration": 0.809686, "end_time": "2022-02-20T01:18:08.303352", "exception": false, "start_time": "2022-02-20T01:18:07.493666", "status": "completed"} tags=[]
from collections import Counter
outputs = []
cnt = 0

for user in tqdm(val_users):
    if user not in positive_items_per_user.keys():
        cnt+=1
        outputs.append(popular_items[0:12])
        continue
    
    most_common_items_of_user = [k for k, v in Counter(positive_items_per_user[user]).most_common()]
    
    user_output = list(most_common_items_of_user[:12])
            
    user_output = user_output + list(popular_items[:12 - len(user_output)])
    outputs.append(user_output)
    
print("mAP Score on Validation set:", mapk(val_items, outputs))

# + [markdown] papermill={"duration": 0.030772, "end_time": "2022-02-20T01:18:08.364580", "exception": false, "start_time": "2022-02-20T01:18:08.333808", "status": "completed"} tags=[]
# 0.0225 mAP. Not bad! We'll now create submissions by adjusting the date ranges of the train set.

# + [markdown] papermill={"duration": 0.031049, "end_time": "2022-02-20T01:18:08.426153", "exception": false, "start_time": "2022-02-20T01:18:08.395104", "status": "completed"} tags=[]
# ### Prediction on Test Set

# + papermill={"duration": 67.604141, "end_time": "2022-02-20T01:19:16.060991", "exception": false, "start_time": "2022-02-20T01:18:08.456850", "status": "completed"} tags=[]
data = pd.read_csv("../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv", dtype={'article_id':str})

data["t_dat"] = pd.to_datetime(data["t_dat"])
train = data.loc[data["t_dat"] >= datetime.datetime(2020,9,7)]

positive_items_per_user = train.groupby(['customer_id'])['article_id'].apply(list)

train['pop_factor'] = train['t_dat'].apply(lambda x: 1/np.exp((datetime.datetime(2020,9,23) - x).days))
popular_items_group = train.groupby(['article_id'])['pop_factor'].sum()
_, popular_items = zip(*sorted(zip(popular_items_group, popular_items_group.keys()))[::-1])

# + papermill={"duration": 5.077815, "end_time": "2022-02-20T01:19:21.169972", "exception": false, "start_time": "2022-02-20T01:19:16.092157", "status": "completed"} tags=[]
submission = pd.read_csv("../input/h-and-m-personalized-fashion-recommendations/sample_submission.csv")
submission.head()

# + papermill={"duration": 10.127038, "end_time": "2022-02-20T01:19:31.328975", "exception": false, "start_time": "2022-02-20T01:19:21.201937", "status": "completed"} tags=[]
from collections import Counter
outputs = []
cnt = 0

for user in tqdm(submission['customer_id']):
    if user not in positive_items_per_user.keys():
        cnt+=1
        outputs.append(popular_items[0:12])
        continue
    
    most_common_items_of_user = [k for k, v in Counter(positive_items_per_user[user]).most_common()]
    
    user_output = list(most_common_items_of_user[:12])
            
    user_output = user_output + list(popular_items[:12 - len(user_output)])
    outputs.append(user_output)
    
str_outputs = []
for output in outputs:
    str_outputs.append(" ".join([str(x) for x in output]))

# + papermill={"duration": 13.304614, "end_time": "2022-02-20T01:19:44.689608", "exception": false, "start_time": "2022-02-20T01:19:31.384994", "status": "completed"} tags=[]
submission['prediction'] = str_outputs
submission.to_csv("submissions.csv", index=False)

# + papermill={"duration": 0.070429, "end_time": "2022-02-20T01:19:44.816768", "exception": false, "start_time": "2022-02-20T01:19:44.746339", "status": "completed"} tags=[]
submission.head()

# + papermill={"duration": 0.055848, "end_time": "2022-02-20T01:19:44.930361", "exception": false, "start_time": "2022-02-20T01:19:44.874513", "status": "completed"} tags=[]


StarcoderdataJupyterScriptsDedupFiltered