# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # RNN 2
# Train and verify the two feature RNN

# +
# %load_ext autoreload

# %autoreload 2

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import math
# -

import MyRNN
from sklearn.linear_model import SGDRegressor
from sklearn import metrics


def remove_outlier(data):
    #calculate what counts as outlier
    Q1 = np.quantile(data, 0.25)
    Q3 = np.quantile(data, 0.75)
    IQR = Q3 - Q1
    
    data_without_fliers = np.copy(data)
    data_without_fliers[data > Q3 + 1.5*IQR] = Q3 + 1.5*IQR
    data_without_fliers[data < Q1 - 1.5*IQR] = Q1 - 1.5*IQR
    
    return data_without_fliers


df_val = pd.read_pickle("df_test_final.pkl")

df_val

# results_df = pd.DataFrame.copy(df_val)
results_df = df_val.drop(['max_days', 
                          'min_days',
                          'days_since',
                          't_5',
                          't_4',
                          't_3',
                          't_2',
                          't_1'], axis = 1)
results_df

# +
#create varaibles to store the results
y_rrn1 = []

r_regres = []
# -

df_val.reset_index(drop=True)

# +
#get the lenght
original_df_len = df_val.shape[0]

#two feature RRN
y_rrn2 = []
rnn_row_id = []
i = 0
for index, row in df_val[80000:].iterrows():
    
    
    print ("progress : " , (index/original_df_len)*100)
    
    if i%10000 == 0:    
        print ("yee")
        name = "out_rnn2_" + str(i) + ".pkl"
        name1 = "out_row_" + str(i) + ".pkl"
        pd.DataFrame(y_rrn2).to_pickle(name)
        pd.DataFrame(rnn_row_id).to_pickle(name1)
    i +=1
    
    #read the data
    x1 = np.array(row.product_seq[:-1])
    x2 = np.array(row.days_since_seq)[1:-1]
    x = np.array([x1,x2]).T
    
    #scale the data
    scale_factor = x.max()
    x_two_features = x/scale_factor
    
    #train the RNN
    my_model = MyRNN.two_feature_RNN()
    my_model.train(remove_outlier(x_two_features), 
                   n_epochs = 1000, lr= 0.01, weight_decay = 0.005)

    #predict the future
    predictions = my_model.predict(x_two_features)
    
    y_rrn2.append(predictions[-1]*scale_factor)
    rnn_row_id.append([row.user_id, row.product_id])
#     y = np.array([y1,y2]).T
    
#     print (index)
#     print (row)
# -

results_df[results_df['rnn_2'] != NaN]

results_df['rnn_2'][num_done:] = y_rrn2

results_df[results_df.rnn_2.notnull()]

results_df.to_pickle("rnn2_backup2.pkl")

StarcoderdataJupyterScriptsDedupFiltered