# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd
from datetime import datetime,timedelta
import numpy as np
import matplotlib.pyplot as plt
import xgboost 
from sklearn.metrics import explained_variance_score
from sklearn import metrics
from sklearn.grid_search import GridSearchCV

pd.options.display.max_rows = 300
pd.options.display.max_columns = 100
# -

data = pd.read_csv('preprocessed_train.csv')

data[(data['scheduledArrival'].isnull())&(data['actualArrival'].isnull())]

data[(data['scheduledDeparture'].isnull())&(data['actualDeparture'].isnull())]

data[(data['scheduledDeparture'].isnull())&(data['scheduledArrival'].isnull())]

data.scheduledArrival[0]

data['runDate']= pd.to_datetime(data['runDate'])
data['scheduledArrival']= pd.to_datetime(data['scheduledArrival'])
data['scheduledDeparture']= pd.to_datetime(data['scheduledDeparture'])
data['actualArrival']= pd.to_datetime(data['actualArrival'])
data['actualDeparture']= pd.to_datetime(data['actualDeparture'])

type(data.scheduledArrival[0])==pd._libs.tslibs.nattype.NaTType


# ## Correcting values in scheduled departure column

def correct_scheduled_arrival(x):
    if (type(x[0])==pd._libs.tslibs.nattype.NaTType) or (type(x[1])==pd._libs.tslibs.nattype.NaTType):
        return x[0]
    
    elif x[0] > x[1]:
        x[0] = x[0] - timedelta(days=1)
    return x[0]


data['scheduledArrival'] = data[['scheduledArrival','scheduledDeparture']].apply(lambda x:correct_scheduled_arrival(x),axis=1)

data[data.scheduledArrival>data.scheduledDeparture]

data.isnull().sum()

data[data.actualDeparture.isnull()]

data.head(100)


# +
#function for checking delay values
def check_delay_values(x):
    if type(x[1])==pd._libs.tslibs.nattype.NaTType:
        return 0
    else:
        return x[0]

data['previous_arrival_delay'] = data.groupby(['runDate','trainCode'])['ArrivalDelay'].shift(fill_value=0)
data['previous_departure_delay'] = data.groupby(['runDate','trainCode'])['DepartureDelay'].shift(fill_value=0)


data['previous_departure_delay'] = data[['previous_departure_delay','scheduledDeparture']].apply(lambda x: check_delay_values(x),axis=1)

data['arrival_delay_for_each_stop'] = data['ArrivalDelay']-data['previous_arrival_delay']
data['departure_delay_for_each_stop'] = data['DepartureDelay']-data['previous_departure_delay']
# -

data.head()


# ## Treating null values in scheduledArrival, scheduledDeparture and actualDeparture columns

def null_value_imputation(x):
    if type(x[0])==pd._libs.tslibs.nattype.NaTType:
        x[0] = x[1] + timedelta(days=x[2])
        return x[0]
    else:
        return x[0]


# +
data['scheduledArrival'] = data[['scheduledArrival','runDate','dayCount']].apply(lambda x: null_value_imputation(x),axis=1)

data['scheduledDeparture'] = data[['scheduledDeparture','runDate','dayCount']].apply(lambda x: null_value_imputation(x),axis=1)
# -

data.head()

# +
# data['actualDeparture'] = data[['actualDeparture','actualArrival']].apply(lambda x: null_value_imputation(x),axis=1)
# -

data.isnull().sum()

data[data.actualDeparture.isnull()]

data['actualDeparture'] = data[['actualDeparture','runDate','dayCount']].apply(lambda x: null_value_imputation(x),axis=1)

data[data.actualDeparture.isnull()]

data.dropna(inplace=True)
data.reset_index(inplace=True,drop=True)

data.runDate.min()

data.runDate.max()

data['scheduledArrival_delta'] =  (data['scheduledArrival']-data['runDate']).apply(lambda x: x.total_seconds() / 60)
data['scheduledDeparture_delta'] =  (data['scheduledDeparture']-data['runDate']).apply(lambda x: x.total_seconds() / 60)
data['actualArrival_delta'] =  (data['actualArrival']-data['runDate']).apply(lambda x: x.total_seconds() / 60)
data['actualDeparture_delta'] =  (data['actualDeparture']-data['runDate']).apply(lambda x: x.total_seconds() / 60)

# +
# data['Stoppage_time'] = (data['scheduledDeparture']-data['scheduledArrival']).apply(lambda x: x.total_seconds() / 60)
# -

data

data.columns



data2 = data.copy()

data2.sort_values(ascending=[True,True,True],by=['runDate','trainCode','distance'],inplace=True)

station_dummies = pd.get_dummies(data2.stations)
station_dummies

trainCode_dummies = pd.get_dummies(data2.trainCode)
trainCode_dummies

data2 = pd.concat([data2,station_dummies,trainCode_dummies],axis=1)
data2

#not going to use standard train_test_split beacause i want to split data frame based on run date
train_size = int(len(data2) * 0.75)
test_size = len(data2) - train_size
train, test = data2.iloc[0:train_size], data2.iloc[train_size:len(data2)]
print(train.shape, test.shape) 

# +
# train_run_time_df = data2.groupby(['trainCode','runDate']).agg({'scheduledDeparture':'min','scheduledArrival':'max'})
# train_run_time_df['train_run_time'] =  (train_run_time_df['scheduledArrival'] - train_run_time_df['scheduledDeparture']).apply(lambda x: x.total_seconds() / 60)
# train_run_time_df.reset_index(inplace=True)
# train_run_time_df

# +
# train_run_time_df.train_run_time.unique()
# -

data2.dayCount.value_counts()

# +
# train_wise_avg_delay = train.groupby(['trainCode','stations']).agg({'arrival_delay_for_each_stop':'mean','departure_delay_for_each_stop':'mean'})
# train_wise_avg_delay.columns=['train_wise_avg_arrival_delay', 'train_wise_avg_departure_delay']
# train_wise_avg_delay.reset_index(inplace=True)
# train_wise_avg_delay

# +
# station_wise_avg_delay = train.groupby(['stations']).agg({'ArrivalDelay':'mean','DepartureDelay':'mean'})
# station_wise_avg_delay.columns=['station_wise_avg_arrival_delay', 'station_wise_avg_departure_delay']
# station_wise_avg_delay.reset_index(inplace=True)
# station_wise_avg_delay

# +
# train = pd.merge(train,train_run_time_df[['trainCode','runDate','train_run_time']],on=['trainCode','runDate'],how='left')
# train = pd.merge(train,train_wise_avg_delay,on=['trainCode','stations'],how='left')
# # train = pd.merge(train,station_wise_avg_delay,on=['stations'],how='left')

# +
# test = pd.merge(test,train_run_time_df[['trainCode','runDate','train_run_time']],on=['trainCode','runDate'],how='left')
# test = pd.merge(test,train_wise_avg_delay,on=['trainCode','stations'],how='left')
# # test = pd.merge(test,station_wise_avg_delay,on=['stations'],how='left')
# -

train.drop(columns=['trainStationId','scheduledArrival','scheduledDeparture','actualArrival','actualDeparture'],inplace=True)
train

test.drop(columns=['trainStationId','scheduledArrival','scheduledDeparture','actualArrival','actualDeparture'],inplace=True)
test

data.corr()

# # Arrival Model

data3 = data.copy()

data3.head()

data3['trainCode'].unique()

trainCode_encoding = {data3['trainCode'].unique()[i]: i+1 for i in range(len(data3['trainCode'].unique()))}
trainCode_encoding

stations_encoding = {data3['stations'].unique()[i]: i+1 for i in range(len(data3['stations'].unique()))}
stations_encoding

data3["trainCode"] = data3["trainCode"].apply(lambda x: trainCode_encoding[x])
data3["trainCode"]

data3["stations"] = data3["stations"].apply(lambda x: stations_encoding[x])
data3["stations"]

data3.columns

feature_cols = ['stations', 'trainCode', 'distance', 'dayCount','scheduledArrival_delta', 
                'scheduledDeparture_delta',
               ]

#not going to use standard train_test_split beacause i want to split data frame based on run date
train_size = int(len(data3) * 0.75)
test_size = len(data3) - train_size
train, test = data3.iloc[0:train_size], data3.iloc[train_size:len(data3)]
print(train.shape, test.shape) 

# +
X_train, y_train = train[feature_cols], train['ArrivalDelay']

X_test, y_test = test[feature_cols], test['ArrivalDelay']

# +


# Parameter Tuning
model = xgboost.XGBRegressor()
param_dist = {"max_depth": [5,10,15],
              "min_child_weight" : [1,3,6],
              "n_estimators": [100,200,300],
              "learning_rate": [0.05, 0.03,0.01],}

grid_search = GridSearchCV(model, param_grid=param_dist, cv = 3, 
                                   verbose=10, n_jobs=-1)
# -

grid_search.fit(X_train, y_train)

grid_search.best_estimator_

arrival_xgb = xgboost.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,
       colsample_bynode=0.7, colsample_bytree=0.7, gamma=0.25, gpu_id=-1,
       importance_type='gain', interaction_constraints='',
       learning_rate=0.03, max_delta_step=0, max_depth=6,
       min_child_weight=7, missing=np.nan, monotone_constraints='()',
       n_estimators=100, n_jobs=8, num_parallel_tree=1,
       objective='reg:squarederror', random_state=0, reg_alpha=0.8,
       reg_lambda=0.8, scale_pos_weight=1, subsample=0.5,
       tree_method='exact', validate_parameters=1, verbosity=None)

arrival_xgb.fit(X_train,y_train)

from sklearn.metrics import mean_squared_error
import math
print(mean_squared_error(y_test, y_pred))
print(math.sqrt(mean_squared_error(y_test, y_pred)))

from sklearn.metrics import r2_score
r2_score(y_test, y_pred)

# # Departure Model

X_train, y_train = train[feature_cols], train['DepartureDelay']
X_test, y_test = test[feature_cols], test['DepartureDelay']

delay_xgb = xgboost.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,
       colsample_bynode=0.7, colsample_bytree=0.7, gamma=0.25, gpu_id=-1,
       importance_type='gain', interaction_constraints='',
       learning_rate=0.03, max_delta_step=0, max_depth=6,
       min_child_weight=7, missing=np.nan, monotone_constraints='()',
       n_estimators=100, n_jobs=8, num_parallel_tree=1,
       objective='reg:squarederror', random_state=0, reg_alpha=0.5,
       reg_lambda=1, scale_pos_weight=1, subsample=0.5,
       tree_method='exact', validate_parameters=1, verbosity=None)

delay_xgb.fit(X_train,y_train)

from sklearn.metrics import mean_squared_error
import math
print(mean_squared_error(y_test, y_pred))
print(math.sqrt(mean_squared_error(y_test, y_pred)))

from sklearn.metrics import r2_score
r2_score(y_test, y_pred)



# +
#Predicting on provided test dataset 
# -

test_df = pd.read_csv('test.csv')

test_df.head()

test_df.isna().sum()

test_df[(test_df.scheduledArrival.isnull())|(test_df.scheduledDeparture.isnull())]

test_df['runDate']= pd.to_datetime(test_df['runDate'])
test_df['scheduledArrival']= pd.to_datetime(test_df['scheduledArrival'])
test_df['scheduledDeparture']= pd.to_datetime(test_df['scheduledDeparture'])

test_df['scheduledArrival'] = test_df[['scheduledArrival','scheduledDeparture']].apply(lambda x:correct_scheduled_arrival(x),axis=1)

test_df['trainCode'] = test_df['trainCode'].apply(lambda x: "T-"+str(x))

test_df

# +
test_df['scheduledArrival'] = test_df[['scheduledArrival','runDate','dayCount']].apply(lambda x: null_value_imputation(x),axis=1)

test_df['scheduledDeparture'] = test_df[['scheduledDeparture','runDate','dayCount']].apply(lambda x: null_value_imputation(x),axis=1)
# -

test_df['scheduledArrival_delta'] =  (test_df['scheduledArrival']-test_df['runDate']).apply(lambda x: x.total_seconds() / 60)
test_df['scheduledDeparture_delta'] =  (test_df['scheduledDeparture']-test_df['runDate']).apply(lambda x: x.total_seconds() / 60)

test_df.trainCode.unique()

test_df["trainCode"] = test_df["trainCode"].apply(lambda x: trainCode_encoding[x])


test_df["stations"] = test_df["stations"].apply(lambda x: stations_encoding[x])

feature_cols = ['stations', 'trainCode', 'distance', 'dayCount','scheduledArrival_delta', 'scheduledDeparture_delta']

test_df[feature_cols]

arrival_delay = arrival_xgb.predict(test_df[feature_cols])

test_df['arrival_delay'] = arrival_delay

test_df

departure_delay = delay_xgb.predict(test_df[feature_cols])

test_df['departure_delay'] = departure_delay

test_df.to_csv('Arnav_.csv',index=False)







StarcoderdataJupyterScriptsDedupFiltered