# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
#import os
#os.environ["MODIN_ENGINE"] = "dask"

import numpy as np
np.random.seed(1337)

import pandas as pd
from matplotlib.pyplot import *
from datetime import datetime
import networkx as nx

matplotlib.pyplot.style.use('classic')

# !python --version

# +
sp500 = pd.read_csv('^GSPC.csv', header = 0, index_col = 'Date')
sp500.index = pd.to_datetime(sp500.index, format = '%d-%m-%y')
sp500 = sp500[1:]
#sp500 = sp500.resample('W').mean()
#sp500.head()
print(len(sp500))

#import nifty50 data
nifty = pd.read_csv('^NSEI.csv', header = 0, index_col = 'Date')
nifty.index = pd.to_datetime(nifty.index, format = '%d-%m-%y')
nifty = nifty.reindex(index = sp500.index, method = 'bfill')
nifty.fillna(method = 'bfill', inplace=True)
#nifty = nifty.resample('W').mean()
#nifty.head()
print(len(nifty))

sing_sti = pd.read_csv('^sti_d.csv', header = 0, index_col = 'Date')
sing_sti.index = pd.to_datetime(sing_sti.index, format = '%Y-%m-%d')
sing_sti = sing_sti.reindex(index = sp500.index, method = 'bfill')
sing_sti.fillna(method = 'bfill', inplace=True)
print(len(sing_sti))

uk_100 = pd.read_csv('^ukx_d.csv', header = 0, index_col = 'Date')
uk_100.index = pd.to_datetime(uk_100.index, format = '%Y-%m-%d')
uk_100 = uk_100.reindex(index = sp500.index, method = 'bfill')
uk_100.fillna(method = 'bfill', inplace=True)
print(len(uk_100))

hangseng = pd.read_csv('^hsi_d.csv', header = 0, index_col = 'Date')
hangseng.index = pd.to_datetime(hangseng.index, format = '%Y-%m-%d')
hangseng = hangseng.reindex(index = sp500.index, method = 'bfill')
hangseng.fillna(method = 'bfill', inplace=True)
print(len(hangseng))

nikkei = pd.read_csv('^nkx_d.csv', header = 0, index_col = 'Date')
nikkei.index = pd.to_datetime(nikkei.index, format = '%Y-%m-%d')
nikkei = nikkei.reindex(index = sp500.index, method = 'bfill')
nikkei.fillna(method = 'bfill', inplace=True)
print(len(nikkei))

shanghai_comp = pd.read_csv('^shc_d.csv', header = 0, index_col = 'Date')
shanghai_comp.index = pd.to_datetime(shanghai_comp.index, format = '%Y-%m-%d')
shanghai_comp = shanghai_comp.reindex(index = sp500.index, method = 'bfill')
shanghai_comp.fillna(method = 'bfill', inplace=True)
print(len(shanghai_comp))

inr = pd.read_csv('DEXINUS.csv', header = 0, index_col = 'DATE')
inr.index = pd.to_datetime(inr.index, format = '%Y-%m-%d')
inr = inr.reindex(index = sp500.index, method = 'bfill')
inr.fillna(method = 'bfill', inplace=True)
print(len(inr))

cny = pd.read_csv('DEXCHUS.csv', header = 0, index_col = 'DATE')
cny.index = pd.to_datetime(cny.index, format = '%Y-%m-%d')
cny = cny.reindex(index = sp500.index, method = 'bfill')
cny.fillna(method = 'bfill', inplace=True)
print(len(cny))

jpy = pd.read_csv('DEXJPUS.csv', header = 0, index_col = 'DATE')
jpy.index = pd.to_datetime(jpy.index, format = '%Y-%m-%d')
jpy = jpy.reindex(index = sp500.index, method = 'bfill')
jpy.fillna(method = 'bfill', inplace=True)
print(len(jpy))

sgd = pd.read_csv('DEXSIUS.csv', header = 0, index_col = 'DATE')
sgd.index = pd.to_datetime(sgd.index, format = '%Y-%m-%d')
sgd = sgd.reindex(index = sp500.index, method = 'bfill')
sgd.fillna(method = 'bfill', inplace=True)
print(len(sgd))

hkd = pd.read_csv('DEXHKUS.csv', header = 0, index_col = 'DATE')
hkd.index = pd.to_datetime(hkd.index, format = '%Y-%m-%d')
hkd = hkd.reindex(index = sp500.index, method = 'bfill')
hkd.fillna(method = 'bfill', inplace=True)
print(len(hkd))

gbp = pd.read_csv('DEXUSUK.csv', header = 0, index_col = 'DATE')
gbp.index = pd.to_datetime(gbp.index, format = '%Y-%m-%d')
gbp = gbp.reindex(index = sp500.index, method = 'bfill')
gbp.fillna(method = 'bfill', inplace=True)
print(len(gbp))
# -

inr.iloc[:, 0] = pd.to_numeric(inr.iloc[:, 0].replace({'.':'0'}))
cny.iloc[:, 0] = pd.to_numeric(cny.iloc[:, 0].replace({'.':'0'}))
jpy.iloc[:, 0] = pd.to_numeric(jpy.iloc[:, 0].replace({'.':'0'}))
sgd.iloc[:, 0] = pd.to_numeric(sgd.iloc[:, 0].replace({'.':'0'}))
hkd.iloc[:, 0] = pd.to_numeric(hkd.iloc[:, 0].replace({'.':'0'}))
gbp.iloc[:, 0] = pd.to_numeric(gbp.iloc[:, 0].replace({'.':'0'}))

gbp = 1/gbp

# +
df = pd.DataFrame(index = sp500.index)
df['nifty'] = nifty['Close']
df['sing_sti'] = sing_sti['Close']
df['hangseng'] = hangseng['Close']
df['nikkei'] = nikkei['Close']
df['shanghai_comp'] = shanghai_comp['Close']
df['sp500'] = sp500['Close']
df['uk_100'] = uk_100['Close']
df = df.transpose()

df_1 = pd.DataFrame(index = sp500.index)
df_1['inr'] = inr
df_1['sgd'] = sgd
df_1['hkd'] = hkd
df_1['jpy'] = jpy
df_1['cny'] = cny
df_1['gbp'] = gbp
df_1['usd'] = 1
df_1 = df_1.transpose()
# -

df_1['base'] = 'usd'
df_1 = df_1.reset_index()
df_exp = df_1.set_index(['index', 'base'])

df_exp = df_exp.reset_index()
df_exp.set_index(['index', 'base'], inplace = True)
df_exp

for currency_fix, base_fix in df_exp.index:
    
    for curr, base in df_exp.index[0:7]:
        df_exp.loc[(curr, currency_fix), :] = df_exp.loc[(curr, base), :]/df_exp.loc[(currency_fix, base_fix), :]
        print(curr,base)

df_exp

df['base'] = ['inr', 'sgd', 'hkd', 'jpy', 'cny', 'usd', 'gbp']
df = df.reset_index()
df_index = df.set_index(['index', 'base'])
df_index

# +
for index, base in df_index.index[0:7]:
    for curr, base_curr in df_exp.loc[(slice(None), base), :].index:
        df_index.loc[(index, curr), :] = df_index.loc[(index, base), :]*df_exp.loc[(curr, base_curr), :]
        
df_index
# -



# # Creating edgelist for network (first attempt) 
# ## (Shelved)

df_edges = pd.DataFrame(columns = ['Source', 'Target'])
df_edges.set_index(['Source', 'Target'], inplace = True)
df_edges

'''
WARNING:
Loop runs for 10-15 mins; output file is stored as csv. Request admin
(Working on incorporating modin to the code:for parallel processing)
'''
for i in range(0, len(df_index.columns), 2):
    corr_df = df_index.iloc[:, i:i+20].T.corr()
    k = -1
    for index, row in corr_df.iterrows():
        k = k+1
        if (k//48 == 1):
            print(k//48)
        for j in range(k, len(corr_df.index)):
            df_edges.loc[((str(index[0]) + '/' + str(index[1])), (str(corr_df.index[j][0]) + '/' + str(corr_df.index[j][1]))), df_index.columns[i]] = row[j]

df_edges.to_csv('daily_corr_networkx.csv')

df_edges = df_edges.reset_index()

df_edges.iloc[:, 5]

G = nx.from_pandas_edgelist(df_edges, "Source", "Target", edge_attr = df_edges.columns[6])

type(G)

# ## --------------------------------------------------------------------------------------------------

# # Creating weighted grpahs[Stored in dictionary: indexed using timestamps] (Second Attempt)

df_corr = df_index.iloc[:, 0:20].T.corr()

df_corr

G = nx.from_pandas_adjacency(df_corr)

# +
G_cache = {}

for i in range(0, len(df_index.columns) - 2, 2):
    corr_df = df_index.iloc[:, i:i+20].T.corr()
    G_cache[df_index.columns[i]] = nx.from_pandas_adjacency(corr_df)
# -

G = G_cache[list(G_cache)[10]]

list(G_cache)[10]

G[('nifty', 'inr')][('sp500', 'usd')]

print(nx.info(G))

nx.draw(G)

# ## ---------------------------------------------------------------------------------------------------

# # Community Detection (Asyncronous Fluid communities)

c = nx.algorithms.community.asyn_fluidc(G, k = 10, seed = 1337)

tuple(sorted(comm) for comm in next(c))

# +
import itertools

l = []
for communities in itertools.islice(c, 10):
    l.append(tuple(sorted(comm) for comm in communities))
# -

l[4]

comm_cache = {}
for timestamp in list(G_cache):
    G = G_cache[timestamp]
    comm_iter = nx.algorithms.community.asyn_fluidc(G, k = 10, seed = 1337)
    comm = []
    for communities in itertools.islice(comm_iter, 10):
        comm.append(tuple(sorted(c) for c in communities))
    
    comm_cache[timestamp] = comm

comm_cache[list(comm_cache)[7]]







# ## -----------------------------------------------------------------------------------------------------

StarcoderdataJupyterScriptsDedupFiltered