# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import norm
import matplotlib.pyplot as plt
import datetime
import math
import matplotlib.pyplot as plt
import keras
import pandas as pd
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from keras.layers import *
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split
from keras.callbacks import EarlyStopping

print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))

# # Load Stockmarket Data

sp500 = pd.read_csv("sp500.csv")
spy = pd.read_csv("spy.csv")

sp500.head()

spy.head()

# ## Verify that the SPY ETF is correlated with the S&P 500
# We verify that the data sets are correlated, and as expected we find all features to be perfectly correlated, with exception to volume.

sp500.corrwith(spy)

# ## Feature Engineering

data = spy

data['d1'] = data.close_adj/data.close_adj.shift(1)
data['d2'] = data.close_adj/data.close_adj.shift(2)
data['d3'] = data.close_adj/data.close_adj.shift(3)

data = data.drop([0,1,2])

data.isna().sum()

data

print ("Average 1 day change",data['d1'].mean())
print ("Average 2 day change",data['d2'].mean())
print ("Average 3 day change",data['d3'].mean())


def plot_dist(data, detail=None, n = 100, s=3, ):
    vars_name = data.name
    mu, std = norm.fit(data)
    plt.hist(data, bins = n, density=True, alpha=0.5, color='b')
    xmin, xmax = plt.xlim()
    x = np.linspace(
        start = xmin,
        stop = xmax,
        num = 100)
    p = norm.pdf(x, mu, std)
    plt.plot(x, p, 'k', linewidth=2)
    for i in range(1, s+1):
        for j in [1, -1]:
            plt.axvline(
                x = mu + i * j * std, 
                linewidth=.5, 
                color="g",
                linestyle = "-.")
    plt.axvline(x=mu, linewidth=.5, color="r")
    plt.xlim(mu - s*std, mu + s*std)
    plt.legend([f'Mu: {mu:.5f}', f'Std: {std:.5f}'])
    title = f'Distribution of {vars_name}'
    if detail != None:
        title = title+'\n'+detail
    plt.title(title)
  
    return plt.show()


plot_dist(data['d1'], "Change over 1 Days")

plot_dist(data['d2'], "Change over 2 Days")

plot_dist(data['d3'], "Change over 3 Days")


# + tags=[]
def calculate_z(column):
    return (column - column.mean())/np.std(column)


# -

np.std(data['d1'])

data['d1_z'] = calculate_z(data['d1'])
data['d2_z'] = calculate_z(data['d2'])
data['d3_z'] = calculate_z(data['d3'])

plot_dist(data['d1_z'], "Z-Score for Single Day Change")

plot_dist(data['d2_z'], "Z-Score for Two Day Change")

plot_dist(data['d3_z'], "Z-Score for Three Day Change")

# ### Day Week Month

data['date'] = pd.to_datetime(data['date'])

data['day'] = data['date'].dt.day_name()

data['month'] = data['date'].dt.month_name()

data

# # Baseline Model

drop_columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'd1_z',"d2_z","d3_z"]
data_prepared = data.drop(columns=drop_columns)

data_prepared = pd.get_dummies(data_prepared)

columns = data_prepared.columns

scaler = MinMaxScaler()
data_prepared[columns] = scaler.fit_transform(data_prepared[columns])

data_prepared['output'] = list(zip(data_prepared['d1'], data_prepared['d2'], data_prepared['d3']))


data_prepared['output'] = data_prepared['d1']
data_d1 = data_prepared.drop(columns=['d1','d2','d3'])
data_d1

data_zipped = data_prepared.drop(columns=['d1','d2','d3'])
data_zipped

# ## Create Training Packets

final = data_d1

records = np.array(final)
records.shape


def history(data, days, output):
    X = []
    y = np.array(data[days:,output])
    for i in range(days, len(data)):
        X.append(data[i-60:i])
    X = np.array(X)
    #X = np.reshape(X, (X.shape[0], X.shape[1], 1))
    return X,y


features, predictors = history(records, 60, 18)

print (f'Inputs {features.shape}')
print (f'Outputs {predictors.shape}')

# + [markdown] tags=[]
# ## Split Training/Test
# -

split = int(len(features)*.74)
print (split)
split = 1792

X_training = features[:split]
y_training = predictors[:split]
X_test = features[split:]
y_test = predictors[split:]
print (f"Training Set: {len(X_training)}, {len(y_training)}")
print (f"Test Set: {len(X_test)}, {len(y_test)}")

y_training

# + [markdown] tags=[]
# ## Model
# -

X_training.shape[1],X_training.shape[2]

model = Sequential()
model.add(LSTM(units = 50, return_sequences = True, input_shape = (X_training.shape[1], X_training.shape[2])))
model.add(Dropout(0.2))
model.add(LSTM(units = 50, return_sequences = True))
model.add(Dropout(0.2))
model.add(LSTM(units = 50, return_sequences = True))
model.add(Dropout(0.2))
model.add(LSTM(units = 50))
model.add(Dropout(0.2))
model.add(Dense(units = 1))

model.summary()

model.compile(optimizer = 'adam', loss = 'mean_squared_error')

callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5)

# Fitting the RNN to the Training set
history = model.fit(
    X_training, 
    y_training, 
    epochs = 10000, 
    batch_size = 1,
    shuffle = False,
    validation_split = .3,
    callbacks = [callback])

len(y_test)

results = model.evaluate(X_test, y_test)
print (results)

predicted_change = model.predict(X_test)

predicted_change

plt.plot(data.loc[800:, ‘Date’],dataset_test.values, color = ‘red’, label = ‘Real TESLA Stock Price’)
plt.plot(df.loc[800:, ‘Date’],predicted_change, color = ‘blue’, label = ‘Predicted TESLA Stock Price’)
plt.xticks(np.arange(0,459,50))
plt.title('TESLA Stock Price Prediction')
plt.xlabel('Time')
plt.ylabel('TESLA Stock Price')
plt.legend()
plt.show()

len(data)

data

test_scaled.shape

X_train = []
y_train = []

engineer_memory(data, history, 

StarcoderdataJupyterScriptsDedupFiltered