# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
from os import path
from CSVUtils import *

# +
rootDir = "./data"
df_namelist = ["^GSPC.csv", "^GDAXI.csv", "^HSI.csv"]
train_df_list = []
test_df_list = []
trainStartDate = pd.to_datetime("2004-01-01")
trainEndDate = pd.to_datetime("2013-12-31")

testStartDate = pd.to_datetime("2015-01-01")
testEndDate = pd.to_datetime("2019-12-31")
for name in df_namelist:
    df = csv2df(rootDir, name, source = "yahoo")
    df = df.sort_values('Date').dropna()
    df = df.reset_index(drop=True)

    train_df = df[(df['Date'] >= trainStartDate) & (df['Date'] <= trainEndDate)]
    test_df = df[(df['Date'] >= testStartDate) & (df['Date'] <= testEndDate)]

    train_df_list.append(train_df)
    test_df_list.append(test_df)
# -

train_df_list[2]

# +
window_size = 6
self_df_list = []

df_list[0].dropna(inplace = True)
intersect_dates = train_df_list[0]['Date']
for df in train_df_list[1:]:
    df.dropna(inplace = True)
    intersect_dates = np.intersect1d(intersect_dates, df['Date'])
# Remove all NAN in the df

start_date = np.min(intersect_dates)
end_date = np.max(intersect_dates)

for i, df in enumerate(df_list):
    self_df_list.append(df[df['Date'].isin(intersect_dates)].reset_index(drop=True))
    print (str(i)+"- Date Equals: "+str(np.array_equal(intersect_dates, self_df_list[i]['Date'])))
    print(str(i)+"- Has NaN: "+str(self_df_list[0].isna().values.any()))


# +
market_number = len(self_df_list)+1
lower_bond = [[0.0]*market_number]*3
lower_bond = np.array(lower_bond)
lower_bond = np.reshape(lower_bond, (1,-1))

upper_bond = [[1.0]*market_number]*3
upper_bond = np.array(upper_bond)
upper_bond = np.reshape(upper_bond, (1,-1))
# -

lower_bond

upper_bond

df.corr().apply(lambda x: x.sort_values(ascending=False))





# +
rootDir = "./data"
df_namelist = ["^BVSP_new.csv", "^TWII_new.csv", "^IXIC_new.csv"]
train_df_list = []
test_df_list = []
trainStartDate = pd.to_datetime("2004-01-01")
trainEndDate = pd.to_datetime("2013-12-31")

testStartDate = pd.to_datetime("2015-01-01")
testEndDate = pd.to_datetime("2019-12-31")
for name in df_namelist:
    df = csv2df(rootDir, name, source = "done")
    df = df.sort_values('Date').dropna()
    df = df.reset_index(drop=True)

    train_df = df[(df['Date'] >= trainStartDate) & (df['Date'] <= trainEndDate)]
    test_df = df[(df['Date'] >= testStartDate) & (df['Date'] <= testEndDate)]

    train_df_list.append(train_df)
    test_df_list.append(test_df)
# -

train_df



StarcoderdataJupyterScriptsDedupFiltered