# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [default]
#     language: python
#     name: python3
# ---

# +
import numpy as np
import tensorflow as tf

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import LabelEncoder
import json
import pickle
import pandas as pd
from sklearn.externals import joblib

# %pylab inline
# -

# # Get data

# ! mkdir ./tmp

# ! wget http://files.grouplens.org/datasets/movielens/ml-100k.zip -O ./ml-100k.zip

# ! unzip -o ./ml-100k.zip

# ! cat ./ml-100k/README

#remove broken symbols
# ! iconv -f utf-8 -t utf-8 -c ml-100k/u.item >  ml-100k/u.item2

# # user part

# ! head -3 ./ml-100k/u.user

df_user = pd.read_csv('./ml-100k/u.user', sep='|', names='user id | age | gender | occupation | zip code'.split(' | '))
df_user['living_area'] = df_user['zip code'].map(lambda x: x[0])
del df_user['zip code']
df_user.head()

res = []
for age in list(map(str, df_user['age'].values)):
    res.append(int(round(int(age), -1)))
df_user['age'] = res

for f in ['age', 'gender', 'occupation', 'living_area']:
    print(f)
    print(df_user[f].nunique())
    print('----')

# +
features_list = ['age', 'gender', 'occupation', 'living_area']
s_users = []
le = LabelEncoder()

users_mat = []
for feature in features_list:
    col = le.fit_transform(df_user[feature].values)
    users_mat.append(col)
    s_users.append(len(le.classes_))
users_mat = np.array(users_mat).T
print(users_mat.shape)
# -

users = {}
for i, id in enumerate(df_user['user id'].values):
    users[id] = users_mat[i]

# # item part

# ! head -3 ./ml-100k/u.item2

df_item = pd.read_csv('./ml-100k/u.item2', 
                      sep='|', 
                      names=(['id', 'title', 'release_date', 'video_release_date', 'url'] + 
                             ['g{}'.format(i) for i in range(19)])
                     )
df_item['year'] = df_item['release_date'].map(lambda x: str(x).split('-')[-1])

res = []
for age in list(map(str, df_item['year'].values)):
    if age == 'nan':
        age='1600'
    res.append(int(round(int(age), -1)))
df_item['decade'] = res

for f in ['decade']:
    print(f)
    print(df_item[f].nunique())
    print('----')

# +
features_list = ['decade'] + ['g{}'.format(i) for i in range(19)]
s_item = []

items_mat = []
for feature in features_list:
    col = le.fit_transform(df_item[feature].values)
    items_mat.append(col)
    s_item.append(len(le.classes_))
items_mat = np.array(items_mat).T
print(items_mat.shape)
# -

items = {}
for i, id in enumerate(df_item['id'].values):
    items[id] = items_mat[i]

# # ratings part

# ! head -3 ./ml-100k/u.data

df_data = pd.read_csv('./ml-100k/u.data', 
                      sep='\t', 
                      names='user id | item id | rating | timestamp'.split(' | ')
                     )

df_data['target'] = df_data['rating'] > 4.5
data = df_data[['user id', 'item id']].as_matrix()
target = df_data['target'].values
print('Mean target: {}'.format(np.mean(target==True)))

# split to pos/neg samples
positive_idx = np.where(target==True)[0]
negative_idx = np.where(target!=True)[0]

from sklearn.cross_validation import train_test_split
pos_idx_tr, pos_idx_te = train_test_split(positive_idx, random_state=42, test_size=0.5)
neg_idx_tr, neg_idx_te = train_test_split(negative_idx, random_state=42, train_size=len(pos_idx_tr)) 


def build_matrix(pos_idx, neg_idx):
    rows_user = []
    rows_item = []
    rows_pair = []
    for idx in list(pos_idx) + list(neg_idx):
        u, i = data[idx]
        # values should be 1-based 
        rows_user.append(users[u] + 1)
        rows_item.append(items[i] + 1)
        # u and i already 1-based
        rows_pair.append(data[idx])
    X = np.hstack(map(np.array, [rows_user, rows_pair, rows_item]))
    Y = np.zeros(len(pos_idx) + len(neg_idx))
    Y[:len(pos_idx)] = 1
    perm = np.random.permutation(X.shape[0])
    return X[perm], Y[perm]


# +
n_users = 943
n_items = 1682


X_tr, Y_tr = build_matrix(pos_idx_tr, neg_idx_tr)
X_te, Y_te = build_matrix(pos_idx_te, neg_idx_te)

# sizes of categorical features
s_features = s_users + [n_users, n_items] + s_item 
# -

print('X_tr shape: ', X_tr.shape)
print('X_te shape: ', X_te.shape)
print('Num of features: ', len(s_features))
print('Size of feature space: ', np.prod(s_features))
print('Sizes of features: ', s_features)

# dump to disk
joblib.dump((X_tr, Y_tr, s_features), './tmp/train_categotical.jl')
joblib.dump((X_te, Y_te, s_features), './tmp/test_categorical.jl')

StarcoderdataJupyterScriptsDedupFiltered