<jupyter_start><jupyter_text>**1) Normal Normal** ***1.1a) Scipy normal PDF function*** KL(P || Q) = sum x in X P(x) * log(P(x) / Q(x))Source: [MLM](https://machinelearningmastery.com/divergence-between-probability-distributions/), [Cory TDS](https://towardsdatascience.com/kl-divergence-python-example-b87069e4b810), and [Swayson GitHub](https://gist.github.com/swayson/86c296aa354a555536e6765bbe726ff7)<jupyter_code># p, q are arrays
def _kl_divergence(p, q):
    p = np.asarray(p, dtype=np.float)
    q = np.asarray(q, dtype=np.float)
    return np.sum(np.where(p != 0, p * np.log(p / q), 0))
    # return sum(p[i] * np.log(p[i]/q[i]) for i in range(len(p)))
p = norm.pdf(x, loc=p_loc, scale=p_scale)
q = norm.pdf(x, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_divergence(p, q))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();
sum(p), sum(q)<jupyter_output><empty_output><jupyter_text>***1.1b) Scipy normal PDF function (rewritten here)*** $p(x ; \mu, \sigma) = \frac{1}{\sigma\sqrt{2\pi}}exp(-\frac{1}{2}\frac{(x - \mu)^2}{\sigma^2})$ Source: [Scipy norm pdf](https://github.com/scipy/scipy/blob/v1.5.4/scipy/stats/_continuous_distns.pyL242-L243)<jupyter_code>def _norm_pdf(x, loc=0., scale=1.):
    return np.exp(-((x-loc)**2 / scale**2) / 2.0) / \
           (scale*np.sqrt(2*np.pi))
p = _norm_pdf(x, loc=p_loc, scale=p_scale)
q = _norm_pdf(x, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_divergence(p, q))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***1.1c) Scipy normal log PDF function***<jupyter_code>log_p = norm.logpdf(x, loc=p_loc, scale=p_scale)
log_q = norm.logpdf(x, loc=q_loc, scale=q_scale)

# I guess you cannot put log_p and log_q into KL Divergence right???
plt.title('KL(P||Q) = %1.3f' % _kl_divergence(p, q))
plt.plot(x, log_p)
plt.plot(x, log_q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***1.1d) Scipy normal log PDF function (rewritten here)*** Source: [Scipy norm log pdf](https://github.com/scipy/scipy/blob/v1.5.4/scipy/stats/_continuous_distns.pyL246-L247)<jupyter_code>def _norm_logpdf(x, loc=0., scale=1.):
    return (-((x-loc)**2 / scale**2) / 2.0) - \
           np.log(scale*np.sqrt(2*np.pi))
log_p = _norm_logpdf(x, loc=p_loc, scale=p_scale)
log_q = _norm_logpdf(x, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_divergence(p, q))
plt.plot(x, log_p)
plt.plot(x, log_q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***1.2a) TFP normal PDF function***<jupyter_code>p_dist = tfd.Normal(loc=p_loc, scale=p_scale)
q_dist = tfd.Normal(loc=q_loc, scale=q_scale)
p = p_dist.prob(x)
q = q_dist.prob(x)

plt.title('KL(P||Q) = %1.3f' % p_dist.kl_divergence(q_dist))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***1.2b) TFP normal PDF and KL functions (rewritten here)***<jupyter_code># TFP only has _log_prob written, thus _prob is the exp of that
def _log_prob(x, loc=0., scale=1.):
    scale = tf.convert_to_tensor(scale)
    log_unnormalized = -0.5 * tf.math.squared_difference(
        x / scale, loc / scale)
    log_normalization = tf.constant(
        0.5 * np.log(2. * np.pi), dtype='float32') + tf.math.log(scale)
    return log_unnormalized - log_normalization

def _prob(x, loc=0., scale=1.):
    return tf.exp(_log_prob(x, loc, scale))
# a, b are distributions.normal.Normal
def _kl_normal_normal(a, b, name=None):
  """Calculate the batched KL divergence KL(a || b) with a and b Normal.
  Args:
    a: instance of a Normal distribution object.
    b: instance of a Normal distribution object.
    name: Name to use for created operations.
      Default value: `None` (i.e., `'kl_normal_normal'`).
  Returns:
    kl_div: Batchwise KL(a || b)
  """
  with tf.name_scope(name or 'kl_normal_normal'):
    b_scale = tf.convert_to_tensor(b.scale)  # We'll read it thrice.
    diff_log_scale = tf.math.log(a.scale) - tf.math.log(b_scale)
    return (
        0.5 * tf.math.squared_difference(a.loc / b_scale, b.loc / b_scale) +
        0.5 * tf.math.expm1(2. * diff_log_scale) -
        diff_log_scale)
p_dist = tfd.Normal(loc=p_loc, scale=p_scale)
q_dist = tfd.Normal(loc=q_loc, scale=q_scale)
p = _prob(x, loc=p_loc, scale=p_scale)
q = _prob(x, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_normal_normal(p_dist, q_dist))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();
type(p_dist), type(q_dist)<jupyter_output><empty_output><jupyter_text>***1.2c) TFP normal log PDF function***<jupyter_code>p_dist = tfd.Normal(loc=p_loc, scale=p_scale)
q_dist = tfd.Normal(loc=q_loc, scale=q_scale)
p = p_dist.log_prob(x)
q = q_dist.log_prob(x)

plt.title('KL(P||Q) = %1.3f' % p_dist.kl_divergence(q_dist))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***1.2d) TFP normal log PDF(rewritten here)***<jupyter_code>p_dist = tfd.Normal(loc=p_loc, scale=p_scale)
q_dist = tfd.Normal(loc=q_loc, scale=q_scale)
p = _log_prob(x, loc=p_loc, scale=p_scale)
q = _log_prob(x, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_normal_normal(p_dist, q_dist))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>**2) Student-t Student-t** ***2.1a) Scipy Student-t PDF function*** Currently used the same KL Divegence function as Normal. Not sure if this is correct.<jupyter_code>def _kl_divergence(p, q):
    return sum(p[i] * np.log(p[i]/q[i]) for i in range(len(p)))
    # return np.sum(np.where(p != 0, p * np.log(p / q), 0))
p = t.pdf(x, df=p_df, loc=p_loc, scale=p_scale)
q = t.pdf(x, df=q_df, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_divergence(p, q))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***2.1b) Scipy Student-t PDF function (rewritten here)*** $f(t) = \frac{\Gamma[(v + 1)/2]}{\sqrt{v\pi}\Gamma[v/2]}(1 + t^2/v)^{-(v+1)/2}$Where:$t = \frac{x - \mu}{\sigma}$$v = DoF $$\Gamma = Gamma function $$d = 1D $$ \sum = 1$Source: [Wikipedia](https://en.wikipedia.org/wiki/Multivariate_t-distribution) Source: [Scipy t pdf](https://github.com/scipy/scipy/blob/v1.5.4/scipy/stats/_continuous_distns.pyL5924-L5931)<jupyter_code>#                                gamma((df+1)/2)
# t.pdf(x, df) = ---------------------------------------------------
#                sqrt(pi*df) * gamma(df/2) * (1+x**2/df)**((df+1)/2)
def _t_pdf(x, df=10., loc=0., scale=1.):
    t = np.asarray(((x-loc)/scale)*1.0)
    v = np.asarray(df*1.0)
    Px = np.exp(sc.gammaln((v+1)/2) - sc.gammaln(v/2))
    Px /= np.sqrt(v*np.pi) * (1+(t**2)/v)**((v+1)/2)
    return Px
p = _t_pdf(x, df=p_df, loc=p_loc, scale=p_scale)
q = _t_pdf(x, df=q_df, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_divergence(p, q))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***2.1c) Scipy Student-t log PDF function***<jupyter_code>log_p = t.logpdf(x, df=p_df, loc=p_loc, scale=p_scale)
log_q = t.logpdf(x, df=q_df, loc=q_loc, scale=q_scale)

# I guess you cannot put log_p and log_q into KL Divergence right???
plt.title('KL(P||Q) = %1.3f' % _kl_divergence(p, q))
plt.plot(x, log_p)
plt.plot(x, log_q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***2.1d) Scipy Student-t log PDF function (rewritten here)*** Source: [Scipy t log pdf](https://github.com/scipy/scipy/blob/v1.5.4/scipy/stats/_continuous_distns.pyL5933-L5937)<jupyter_code>def _t_logpdf(x, df=10., loc=0., scale=1.):
    t = ((x-loc)/scale)*1.0
    v = df*1.0
    lPx = sc.gammaln((v+1)/2)-sc.gammaln(v/2)
    lPx -= 0.5*np.log(v*np.pi) + (v+1)/2*np.log(1+(t**2)/v)
    return lPx
log_p = _t_logpdf(x, loc=p_loc, scale=p_scale)
log_q = _t_logpdf(x, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_divergence(p, q))
plt.plot(x, log_p)
plt.plot(x, log_q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***2.2a) TFP Student-t PDF function*** tfp currently does not have an in-house Student-T function. Therefore, we have to create it ourselves...<jupyter_code>p_dist = tfd.StudentT(df=p_df, loc=p_loc, scale=p_scale)
q_dist = tfd.StudentT(df=q_df, loc=q_loc, scale=q_scale)
p = p_dist.prob(x)
q = q_dist.prob(x)

# plt.title('KL(P||Q) = %1.3f' % p_dist.kl_divergence(q_dist))
plt.title('No KL(P||Q)')
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***2.2b) TFP Student-t PDF and KL functions (rewritten here)***<jupyter_code>def _t_log_prob(x, df, loc, scale):
  """
  Compute log probability of Student T distribution.
  Note that scale can be negative.
  Args:
    x: Floating-point `Tensor`. Where to compute the log probabilities.
    df: Floating-point `Tensor`. The degrees of freedom of the
      distribution(s). `df` must contain only positive values.
    loc: Floating-point `Tensor`; the location(s) of the distribution(s).
    scale: Floating-point `Tensor`; the scale(s) of the distribution(s).
  Returns:
    A `Tensor` with shape broadcast according to the arguments.
  """
  # Writing `y` this way reduces XLA mem copies.
  y = (x - loc) * (tf.math.rsqrt(df) / scale)
  log_unnormalized_prob = -0.5 * (df + 1.) * log1psquare(y)
  log_normalization = (
      tf.math.log(tf.abs(scale)) + 0.5 * tf.math.log(df) +
      0.5 * np.log(np.pi) + tfp_math.log_gamma_difference(0.5, 0.5 * df))
  return log_unnormalized_prob - log_normalization

def _t_prob(x, df, loc, scale):
    return tf.exp(_t_log_prob(x, df, loc, scale))

'''
# TFP only has _log_prob written, thus _prob is the exp of that
def _log_prob(x, loc=0., scale=1.):
    scale = tf.convert_to_tensor(scale)
    log_unnormalized = -0.5 * tf.math.squared_difference(
        x / scale, loc / scale)
    log_normalization = tf.constant(
        0.5 * np.log(2. * np.pi), dtype='float32') + tf.math.log(scale)
    return log_unnormalized - log_normalization

def _prob(x, loc=0., scale=1.):
    return tf.exp(_log_prob(x, loc, scale))
'''
def _kl_student_student(a, b, name=None):
    with tf.name_scope(name or 'kl_student_student'):
        b_scale = tf.convert_to_tensor(b.scale)  # We'll read it thrice.
        diff_log_scale = tf.math.log(a.scale) - tf.math.log(b_scale)
        return (
            0.5 * tf.math.squared_difference(a.loc / b_scale, b.loc / b_scale) +
            0.5 * tf.math.expm1(2. * diff_log_scale) -
            diff_log_scale)

'''
def _kl_normal_normal(a, b, name=None):
  """Calculate the batched KL divergence KL(a || b) with a and b Normal.
  Args:
    a: instance of a Normal distribution object.
    b: instance of a Normal distribution object.
    name: Name to use for created operations.
      Default value: `None` (i.e., `'kl_normal_normal'`).
  Returns:
    kl_div: Batchwise KL(a || b)
  """
  with tf.name_scope(name or 'kl_normal_normal'):
    b_scale = tf.convert_to_tensor(b.scale)  # We'll read it thrice.
    diff_log_scale = tf.math.log(a.scale) - tf.math.log(b_scale)
    return (
        0.5 * tf.math.squared_difference(a.loc / b_scale, b.loc / b_scale) +
        0.5 * tf.math.expm1(2. * diff_log_scale) -
        diff_log_scale)
'''
p_dist = tfd.StudentT(df=p_df, loc=p_loc, scale=p_scale)
q_dist = tfd.StudentT(df=q_df, loc=q_loc, scale=q_scale)
p = _t_prob(x, df=p_df, loc=p_loc, scale=p_scale)
q = _t_prob(x, df=p_df, loc=q_loc, scale=q_scale)

# plt.title('KL(P||Q) = %1.3f' % p_dist.kl_divergence(q_dist))
plt.title('KL(P||Q) = %1.3f' % _kl_student_student(p_dist, q_dist))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();

'''
p_dist = tfd.Normal(loc=p_loc, scale=p_scale)
q_dist = tfd.Normal(loc=q_loc, scale=q_scale)
p = _prob(x, loc=p_loc, scale=p_scale)
q = _prob(x, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_normal_normal(p_dist, q_dist))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();
'''<jupyter_output><empty_output><jupyter_text>***2.2c) TFP Student-t log PDF function***<jupyter_code>p_dist = tfd.StudentT(df=p_df, loc=p_loc, scale=p_scale)
q_dist = tfd.StudentT(df=q_df, loc=q_loc, scale=q_scale)
p = p_dist.log_prob(x)
q = q_dist.log_prob(x)

# plt.title('KL(P||Q) = %1.3f' % p_dist.kl_divergence(q_dist))
plt.title('No KL(P||Q)')
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();<jupyter_output><empty_output><jupyter_text>***2.2d) TFP Student-t log PDF (rewritten here)***<jupyter_code>'''
p_dist = tfd.Normal(loc=p_loc, scale=p_scale)
q_dist = tfd.Normal(loc=q_loc, scale=q_scale)
p = _log_prob(x, loc=p_loc, scale=p_scale)
q = _log_prob(x, loc=q_loc, scale=q_scale)

plt.title('KL(P||Q) = %1.3f' % _kl_normal_normal(p_dist, q_dist))
plt.plot(x, p)
plt.plot(x, q, c='red')
plt.show();
'''<jupyter_output><empty_output><jupyter_text>**3) CoupledNormal CoupledNormal**<jupyter_code># TO-DO<jupyter_output><empty_output><jupyter_text>**4) MultivariateCoupledNormal MultivariateCoupledNormal**<jupyter_code># TO-DO<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup