# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
# %matplotlib inline
sns.set()

# ## Prepare Data

# +
size_d = 500 # sample size
var_dim = 5  # dimension of variable.

x = np.array([list(np.random.permutation(np.linspace(-6, 6, size_d))) for _ in range(var_dim)]).T

df_x = pd.DataFrame(data=x, columns=['a', 'b', 'c', 'd', 'e'])
df_x.plot()
plt.show()


# -

def obs_process(x):
    return (5*(x+3)*x*(x-3)).sum()
y_true = np.array([obs_process(_) for _ in df_x.values])
sns.lineplot(x=range(len(y_true)), y=y_true)

# Prepare test data.
np.random.seed(0)
y_noise = y_true + np.random.randn(size_d) * 50.0
size_p = 100
new_x_idx = np.random.permutation(df_x.index)
new_x_idx = new_x_idx[:size_p]
new_x_idx.sort()
df_new_x = df_x.loc[new_x_idx, :]
new_x = df_new_x.values
new_y_true = np.array([obs_process(_) for _ in df_new_x.values])

df_x.loc[:, 'y_true'] = y_true
df_x.loc[:, 'y_noise'] = y_noise
df_x['Predict_objective'] = np.nan
for i, idx in enumerate(new_x_idx):
    df_x.loc[idx, 'Predict_objective'] = new_y_true[i]

# ### This is our data
# a,b,c,d and e is explanatory variables.  
# y_true and y_noise is target variable.

df_x.head()

fig, ax = plt.subplots(2,1, figsize=(6,11))
df_x.loc[:,['a', 'b', 'c', 'd', 'e']].plot(ax=ax[1])
pd.DataFrame(
    {'x': df_x.index,
     'y_true': y_true}).plot(x='x', ax=ax[0], style='-', color='gray')
pd.DataFrame(
    {'x': df_x.index,
     'y_noise': y_noise}).plot(x='x', ax=ax[0], style='.', color='gray')
pd.DataFrame(
    {'x': df_new_x.index,
     'Predict Objective': new_y_true}).plot(x='x', ax=ax[0], style='.', color='r')
ax[0].set_title('y')
ax[1].set_title('x')
plt.show()

# ## Kernel regression with sampled teacher data

# +
np.random.seed(10)

sample_size = 200  # Size of data to be used in learning.
beta = 0.1  # Kernel parameter
theta = 0.1  # Regularize parameter

# Gaussian Kernel
def kernel(x1, x2, beta=beta):
    return np.exp(- beta * np.linalg.norm(x1 - x2))

# Sampling teacher data.
df = df_x.sample(sample_size)
selected_x = df.loc[:, ['a', 'b', 'c', 'd', 'e']].values
selected_y = df.loc[:, 'y_noise'].values

# Gram matrix.
K = np.zeros((sample_size, sample_size))
for i in range(sample_size):
    for j in range(sample_size):
        K[i,j] = kernel(selected_x[i], selected_x[j])
        
# Compute psuede inverse matrix of the gram matrix.
K_pinv = np.linalg.pinv(K + theta*np.identity(sample_size))

# Compute weight of learned model of each data point.
alpha = selected_y@K_pinv

# Weight of new data of each data point. 
w_x = np.zeros((sample_size, size_p))
for i in range(sample_size):
    for j in range(size_p):
        w_x[i, j] = kernel(new_x[j, :], selected_x[i, :])  

# Prediction
y_predict = alpha@w_x

sns.scatterplot(x=new_y_true, y=y_predict)
plt.show()


# -

# ## Predict another data

def myx2(x):
    return np.array([0.12783293, 0.169141  , x , 0.3474041 , 0.4787337 ])
x2 = np.array([myx2(_) for _ in np.arange(-6,6, 0.1)])
y_true2 = np.array([obs_process(_) for _ in x2])
sns.lineplot(x=np.arange(-6,6, 0.1), y=y_true2)
plt.show()

# +
size_p2 = y_true2.shape[0]

# Weight of new data of each data point. 
w_x = np.zeros((sample_size, size_p2))
for i in range(sample_size):
    for j in range(size_p2):
        w_x[i, j] = kernel(x2[j, :], selected_x[i, :])  

# Prediction
y_predict2 = (selected_y.T@K_pinv)@w_x

fig, ax = plt.subplots()
pd.DataFrame(
    {'x': [_[2] for _ in x2],
     'y_true': y_true2}).plot(x='x', ax=ax)
pd.DataFrame(
    {'x': [_[2] for _ in x2],
     'y_predict': y_predict2}).plot(x='x', style='.', ax=ax)
plt.show()

StarcoderdataJupyterScriptsDedupFiltered