# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:py3]
#     language: python
#     name: conda-env-py3-py
# ---

# <img src="http://hilpisch.com/tpq_logo.png" alt="The Python Quants" width="35%" align="right" border="0"><br>

# # Python for Finance

# **Analyze Big Financial Data**
#
# O'Reilly (2014)
#
# <NAME>

# <img style="border:0px solid grey;" src="http://hilpisch.com/python_for_finance.png" alt="Python for Finance" width="30%" align="left" border="0">

# **Buy the book ** |
# <a href='http://shop.oreilly.com/product/0636920032441.do' target='_blank'>O'Reilly</a> |
# <a href='http://www.amazon.com/Yves-Hilpisch/e/B00JCYHHJM' target='_blank'>Amazon</a>
#
# **All book codes & IPYNBs** |
# <a href="http://oreilly.quant-platform.com">http://oreilly.quant-platform.com</a>
#
# **The Python Quants GmbH** | <a href='http://tpq.io' target='_blank'>http://tpq.io</a>
#
# **Contact us** | <a href='mailto:<EMAIL>'><EMAIL></a>

# # Statistics (b)

import seaborn as sns; sns.set()
import matplotlib as mpl
mpl.rcParams['font.family'] = 'serif'

# ## Principal Component Analysis

# + uuid="a82b3a95-9ee0-4ac3-9601-16972b7728b0"
import numpy as np
import pandas as pd
from pandas_datareader import data as web
from sklearn.decomposition import KernelPCA
# -

# ### DAX Index and its 30 Stocks

# + uuid="b5f51ac2-45b2-4a72-befd-27f38ef6f6d8"
symbols = ['ADS.DE', 'ALV.DE', 'BAS.DE', 'BAYN.DE', 'BEI.DE',
           'BMW.DE', 'CBK.DE', 'CON.DE', 'DAI.DE', 'DB1.DE',
           'DBK.DE', 'DPW.DE', 'DTE.DE', 'EOAN.DE', 'FME.DE',
           'FRE.DE', 'HEI.DE', 'HEN3.DE', 'IFX.DE', 'LHA.DE',
           'LIN.DE', 'LXS.DE', 'MRK.DE', 'MUV2.DE', 'RWE.DE',
           'SAP.DE', 'SDF.DE', 'SIE.DE', 'TKA.DE', 'VOW3.DE',
           '^GDAXI']

# + uuid="1556b1fa-99da-4122-8aa7-b9e06c069a21"
# %%time
data = pd.DataFrame()
for sym in symbols:
    data[sym] = web.DataReader(sym, data_source='yahoo')['Close']
data = data.dropna()

# + uuid="b4813521-7846-4660-8391-770fea7c318f"
dax = pd.DataFrame(data.pop('^GDAXI'))

# + uuid="03d4a81f-3bd8-47dd-a5b6-84f060fcdf97"
data[data.columns[:6]].head()
# -

# ### Applying PCA

# + uuid="5ae896a2-8493-4d0c-978c-1b4f26ce2291"
scale_function = lambda x: (x - x.mean()) / x.std()

# + uuid="4503ea73-aa94-4187-b9c8-cbf00564bf7a"
pca = KernelPCA().fit(data.apply(scale_function))

# + uuid="e6f9ebd8-1d61-4047-a254-fe0e524f4c8f"
len(pca.lambdas_)

# + uuid="1694bb38-778a-403f-82af-0129fdd3a260"
pca.lambdas_[:10].round()

# + uuid="8c11b92d-c1ea-48dd-a47a-fb4d980b5f40"
get_we = lambda x: x / x.sum()

# + uuid="b72fcfcb-d6f0-4101-a59b-c8df516154e8"
get_we(pca.lambdas_)[:10]

# + uuid="d40695cd-b736-484d-bc11-4755eb1d3448"
get_we(pca.lambdas_)[:5].sum()
# -

# ### Constructing a PCA Index

# + uuid="b4691f51-fa25-48d1-855d-907136ee39a7"
pca = KernelPCA(n_components=1).fit(data.apply(scale_function))
dax['PCA_1'] = pca.transform(-data)

# + uuid="9addd109-7de2-4998-b2f0-11c72c3b98da"
import matplotlib.pyplot as plt
# %matplotlib inline
dax.apply(scale_function).plot(figsize=(8, 4))
# tag: pca_1
# title: German DAX index and PCA index with 1 component

# + uuid="4b039aad-93a0-439b-b2d5-d24119604fa4"
pca = KernelPCA(n_components=5).fit(data.apply(scale_function))
pca_components = pca.transform(-data)
weights = get_we(pca.lambdas_)
dax['PCA_5'] = np.dot(pca_components, weights)

# + uuid="fc7a90b6-167b-44dc-9a31-38d479be9026"
import matplotlib.pyplot as plt
# %matplotlib inline
dax.apply(scale_function).plot(figsize=(8, 4))
# tag: pca_2
# title: German DAX index and PCA indices with 1 and 5 components

# + uuid="ceb357ad-d98a-4618-8e73-a6dc77a80436"
import matplotlib as mpl
mpl_dates = mpl.dates.date2num(data.index.to_pydatetime())
mpl_dates

# + uuid="60ae390c-9534-4e57-bc1e-42e261d0abc9"
plt.figure(figsize=(8, 4))
plt.scatter(dax['PCA_5'], dax['^GDAXI'], c=mpl_dates)
lin_reg = np.polyval(np.polyfit(dax['PCA_5'],
                                dax['^GDAXI'], 1),
                                dax['PCA_5'])
plt.plot(dax['PCA_5'], lin_reg, 'r', lw=3)
plt.grid(True)
plt.xlabel('PCA_5')
plt.ylabel('^GDAXI')
plt.colorbar(ticks=mpl.dates.DayLocator(interval=250),
                format=mpl.dates.DateFormatter('%d %b %y'))
# tag: pca_3
# title: DAX return values against PCA return values with linear regression

# + uuid="2c9bea6f-bed0-49d0-83ad-a30c17b31922"
cut_date = '2011/7/1'
early_pca = dax[dax.index < cut_date]['PCA_5']
early_reg = np.polyval(np.polyfit(early_pca,
                dax['^GDAXI'][dax.index < cut_date], 1),
                early_pca)

# + uuid="0df5fdce-32ae-4b85-836d-91355f33bb90"
late_pca = dax[dax.index >= cut_date]['PCA_5']
late_reg = np.polyval(np.polyfit(late_pca,
                dax['^GDAXI'][dax.index >= cut_date], 1),
                late_pca)

# + uuid="ad8b2336-b13f-49a2-92b9-3f7b340be327"
plt.figure(figsize=(8, 4))
plt.scatter(dax['PCA_5'], dax['^GDAXI'], c=mpl_dates)
plt.plot(early_pca, early_reg, 'r', lw=3)
plt.plot(late_pca, late_reg, 'r', lw=3)
plt.grid(True)
plt.xlabel('PCA_5')
plt.ylabel('^GDAXI')
plt.colorbar(ticks=mpl.dates.DayLocator(interval=250),
                format=mpl.dates.DateFormatter('%d %b %y'))
# tag: pca_7
# title: DAX index values against PCA index values with early and late regression (regime switch)
# -

# ## Bayesian Regression

# ### Bayes' Formula

# ### PyMC3

# + uuid="72026681-577f-49e6-abe0-f5554e39fa06"
import warnings
warnings.simplefilter('ignore')
import pymc3 as pm
import numpy as np
np.random.seed(1000)
import matplotlib.pyplot as plt
# %matplotlib inline
# -

# ### Introductory Example

# + uuid="4b612fde-e947-4a33-a275-6202d7ded7bb"
x = np.linspace(0, 10, 500)
y = 4 + 2 * x + np.random.standard_normal(len(x)) * 2

# + uuid="ad9f9a77-ffb5-4eed-a79b-bd2ea33ae0e2"
reg = np.polyfit(x, y, 1)
  # linear regression

# + uuid="4f6cbf6d-06ce-4f8d-a7a7-1209004b686a"
plt.figure(figsize=(8, 4))
plt.scatter(x, y, c=y, marker='v')
plt.plot(x, reg[1] + reg[0] * x, lw=2.0)
plt.colorbar()
plt.grid(True)
plt.xlabel('x')
plt.ylabel('y')
# tag: pm_fig_0
# title: Sample data points a regresion line
# size: 90

# + uuid="5903887a-2a6b-4dce-89f1-9de9be686ef4"
reg

# + uuid="46936559-a438-475e-bd5b-f669abed06e2"
with pm.Model() as model: 
        # model specifications in PyMC3
        # are wrapped in a with-statement
    # define priors
    alpha = pm.Normal('alpha', mu=0, sd=20)
    beta = pm.Normal('beta', mu=0, sd=20)
    sigma = pm.Uniform('sigma', lower=0, upper=10)
    
    # define linear regression
    y_est = alpha + beta * x
    
    # define likelihood
    likelihood = pm.Normal('y', mu=y_est, sd=sigma, observed=y)
    
    # inference
    start = pm.find_MAP()
      # find starting value by optimization
    step = pm.NUTS(state=start)
      # instantiate MCMC sampling algorithm
    trace = pm.sample(100, step, start=start, progressbar=False)
      # draw 100 posterior samples using NUTS sampling

# + uuid="e06c4eed-10d4-4d4c-aea0-4e4164f4d7f5"
trace[0]

# + uuid="c172bd48-8b11-4128-9e97-f20ce835d319"
fig = pm.traceplot(trace, lines={'alpha': 4, 'beta': 2, 'sigma': 2})
plt.figure(figsize=(8, 8))
# tag: pm_fig_1
# title: Trace plots for alpha, beta and sigma
# size: 90

# + uuid="99e44b63-168c-415a-be8f-9347f45578aa"
plt.figure(figsize=(8, 4))
plt.scatter(x, y, c=y, marker='v')
plt.colorbar()
plt.grid(True)
plt.xlabel('x')
plt.ylabel('y')
for i in range(len(trace)):
    plt.plot(x, trace['alpha'][i] + trace['beta'][i] * x)
# tag: pm_fig_2
# title: Sample data and regression lines from Bayesian regression
# size: 90
# -

# ### Real Data

# + uuid="0562fc9b-01c8-4eae-8e57-959a6e551610"
import warnings
warnings.simplefilter('ignore')
import pytz
import datetime as dt
# -

data = pd.DataFrame()
symbols = ['GLD', 'GDX']
for sym in symbols:
    data[sym] = web.DataReader(sym, data_source='yahoo')['Adj Close']

# + uuid="a3774b4d-09a8-47ce-a56e-aac894334dbd"
data.info()

# + uuid="07c6dea4-fd4f-496c-86f2-e92689797fe8"
data.plot(figsize=(8, 4))
# tag: zip_fig_1
# title: Co-movements of traiding pair
# size: 90

# + uuid="8b467729-65d9-4579-8b4c-944fb18bee4c"
data.ix[-1] / data.ix[0] - 1

# + uuid="ed244e7b-692f-4e14-b642-80ceccf221b2"
data.corr()

# + uuid="a92a57dd-76c4-4e8b-ab75-8523fbb49435"
data.index

# + uuid="facf1920-22f9-4300-90ed-39f79d9b024c"
import matplotlib as mpl
mpl_dates = mpl.dates.date2num(data.index.to_pydatetime())
mpl_dates

# + uuid="f73eecbd-a135-4448-a5d4-8caa6259039e"
plt.figure(figsize=(8, 4))
plt.scatter(data['GDX'], data['GLD'], c=mpl_dates, marker='o')
plt.grid(True)
plt.xlabel('GDX')
plt.ylabel('GLD')
plt.colorbar(ticks=mpl.dates.DayLocator(interval=250),
             format=mpl.dates.DateFormatter('%d %b %y'))
# tag: zip_fig_2
# title: Scatter plot of prices for GLD and GDX
# size: 90

# + uuid="0cf4e75d-b4ea-4822-9e25-8c837fe6049d"
with pm.Model() as model:
    alpha = pm.Normal('alpha', mu=0, sd=20)
    beta = pm.Normal('beta', mu=0, sd=20)
    sigma = pm.Uniform('sigma', lower=0, upper=50)
    
    y_est = alpha + beta * data['GDX'].values
    
    likelihood = pm.Normal('GLD', mu=y_est, sd=sigma,
                           observed=data['GLD'].values)
    
    start = pm.find_MAP()
    step = pm.NUTS(state=start)
    trace = pm.sample(100, step, start=start, progressbar=False)

# + uuid="1a64e594-4985-4413-b4b6-c8400691fbc2"
fig = pm.traceplot(trace)
plt.figure(figsize=(8, 8))
# tag: zip_fig_3
# title: Trace plots for alpha, beta and sigma based on GDX and GLD data
# size: 90

# + uuid="cd213b2f-72e3-4afb-8be3-60e61e94c7c6"
plt.figure(figsize=(8, 4))
plt.scatter(data['GDX'], data['GLD'], c=mpl_dates, marker='o')
plt.grid(True)
plt.xlabel('GDX')
plt.ylabel('GLD')
for i in range(len(trace)):
    plt.plot(data['GDX'], trace['alpha'][i] + trace['beta'][i] * data['GDX'])
plt.colorbar(ticks=mpl.dates.DayLocator(interval=250),
             format=mpl.dates.DateFormatter('%d %b %y'))
# tag: zip_fig_4
# title: Scatter plot with "simple" regression lines
# size: 90
# -

# ## Conclusions

# ## Further Reading

# <img src="http://hilpisch.com/tpq_logo.png" alt="The Python Quants" width="35%" align="right" border="0"><br>
#
# <a href="http://tpq.io" target="_blank">http://tpq.io</a> | <a href="http://twitter.com/dyjh" target="_blank">@dyjh</a> | <a href="mailto:<EMAIL>"><EMAIL></a>
#
# **Quant Platform** |
# <a href="http://quant-platform.com">http://quant-platform.com</a>
#
# **Python for Finance** |
# <a href="http://python-for-finance.com" target="_blank">Python for Finance @ O'Reilly</a>
#
# **Derivatives Analytics with Python** |
# <a href="http://derivatives-analytics-with-python.com" target="_blank">Derivatives Analytics @ Wiley Finance</a>
#
# **Listed Volatility and Variance Derivatives** |
# <a href="http://lvvd.tpq.io" target="_blank">Listed VV Derivatives @ Wiley Finance</a>
#
# **Python Training** |
# <a href="http://training.tpq.io" target="_blank">Python for Finance University Certificate</a>

StarcoderdataJupyterScriptsDedupFiltered