# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="kAbpkqfJzzyT" colab_type="text"
# # More Regression: RBF BASIS / Modified from : 
#
# 1. https://github.com/jakevdp/PythonDataScienceHandbook 
#
#

# + deletable=true editable=true id="cIRrwEhWzzyV" colab_type="code" colab={}
# %matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
import numpy as np

# + id="tet98sU-F8l-" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 285} executionInfo={"status": "ok", "timestamp": 1599767235389, "user_tz": 240, "elapsed": 716, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "12649994350437867413"}} outputId="8bc7677c-cb1f-4f08-df93-56c78ba54487"
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
rng = np.random.RandomState(1)
x = 10 * rng.rand(50)
y = np.sin(x) + 0.1 * rng.randn(50)
plt.scatter(x, y)



# + [markdown] deletable=true editable=true id="wbEHlS5fzzy6" colab_type="text"
# ### Gaussian basis functions
#
# Of course, other basis functions are possible.
# For example, one useful pattern is to fit a model that is not a sum of polynomial bases, but a sum of Gaussian bases.
# The result might look something like the following figure:

# + [markdown] deletable=true editable=true id="idNV1KQ-zzy8" colab_type="text"
#
#
# These Gaussian basis functions are not built into Scikit-Learn, but we can write a custom transformer that will create them, as shown here and illustrated in the following figure (Scikit-Learn transformers are implemented as Python classes; reading Scikit-Learn's source is a good way to see how they can be created):

# + deletable=true editable=true id="TuqPXzCXzzy8" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 268} executionInfo={"status": "ok", "timestamp": 1599767336030, "user_tz": 240, "elapsed": 762, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "12649994350437867413"}} outputId="cd630ceb-beb0-4b35-9b4c-e3392ee207f1"

from sklearn.base import BaseEstimator, TransformerMixin

class GaussianFeatures(BaseEstimator, TransformerMixin):
    """Uniformly spaced Gaussian features for one-dimensional input"""
    
    def __init__(self, N, width_factor=2.0):
        self.N = N
        self.width_factor = width_factor
    
    @staticmethod
    def _gauss_basis(x, y, width, axis=None):
        arg = (x - y) / width
        return np.exp(-0.5 * np.sum(arg ** 2, axis))
        
    def fit(self, X, y=None):
        # create N centers spread along the data range
        self.centers_ = np.linspace(X.min(), X.max(), self.N)
        self.width_ = self.width_factor * (self.centers_[1] - self.centers_[0])
        return self
        
    def transform(self, X):
        return self._gauss_basis(X[:, :, np.newaxis], self.centers_,
                                 self.width_, axis=1)
    
gauss_model = make_pipeline(GaussianFeatures(20),
                            LinearRegression())
gauss_model.fit(x[:, np.newaxis], y)
xfit = np.linspace(0, 10, 1000)

yfit = gauss_model.predict(xfit[:, np.newaxis])

plt.scatter(x, y)
plt.plot(xfit, yfit)
plt.xlim(0, 10);

# + deletable=true editable=true id="5gJx6akwzzzB" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 272} executionInfo={"status": "ok", "timestamp": 1599767525367, "user_tz": 240, "elapsed": 730, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "12649994350437867413"}} outputId="df193a54-7785-4dd7-cef8-a687f9e63a93"
model = make_pipeline(GaussianFeatures(30),
                      LinearRegression())
model.fit(x[:, np.newaxis], y)

plt.scatter(x, y)
plt.plot(xfit, model.predict(xfit[:, np.newaxis]))

plt.xlim(0, 10)
plt.ylim(-1.5, 1.5);


# + [markdown] deletable=true editable=true id="nBQaS89KzzzE" colab_type="text"
# With the data projected to the 30-dimensional basis, the model has far too much flexibility and goes to extreme values between locations where it is constrained by data.
# We can see the reason for this if we plot the coefficients of the Gaussian bases with respect to their locations:

# + deletable=true editable=true id="BBHQFxAfzzzE" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 285} executionInfo={"status": "ok", "timestamp": 1599767645924, "user_tz": 240, "elapsed": 1136, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "12649994350437867413"}} outputId="107b901e-4f6f-4a6d-fa3b-aaabde722ddc"
def basis_plot(model, title=None):
    fig, ax = plt.subplots(2, sharex=True)
    model.fit(x[:, np.newaxis], y)
    ax[0].scatter(x, y)
    ax[0].plot(xfit, model.predict(xfit[:, np.newaxis]))
    ax[0].set(xlabel='x', ylabel='y', ylim=(-1.5, 1.5))
    
    if title:
        ax[0].set_title(title)

    ax[1].plot(model.steps[0][1].centers_,
               model.steps[1][1].coef_)
    ax[1].set(xlabel='basis location',
              ylabel='coefficient',
              xlim=(0, 10))
    
model = make_pipeline(GaussianFeatures(30), LinearRegression())
basis_plot(model)

# + [markdown] deletable=true editable=true id="q2V9CBwnzzzH" colab_type="text"
# The lower panel of this figure shows the amplitude of the basis function at each location.
# This is typical over-fitting behavior when basis functions overlap: the coefficients of adjacent basis functions blow up and cancel each other out.
# We know that such behavior is problematic, and it would be nice if we could limit such spikes expliticly in the model by penalizing large values of the model parameters.
# Such a penalty is known as *regularization*, and comes in several forms.

# + id="QeLeMqudGVSU" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 285} executionInfo={"status": "ok", "timestamp": 1599759544769, "user_tz": 240, "elapsed": 1634, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "12649994350437867413"}} outputId="36a8052d-0c16-41d3-92cd-a9d938e4a45f"
model = make_pipeline(GaussianFeatures(20), LinearRegression())
basis_plot(model)

# + id="5IiXF59YGZif" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 285} executionInfo={"status": "ok", "timestamp": 1599767747786, "user_tz": 240, "elapsed": 1029, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "12649994350437867413"}} outputId="68a686e0-5a74-430f-9718-3ec2465247fb"
model = make_pipeline(GaussianFeatures(10), LinearRegression())
basis_plot(model)

StarcoderdataJupyterScriptsDedupFiltered