# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np


# +
def gelu1(x):
  cdf = 0.5 * (1.0 + tf.tanh(
    (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))))
  return x * cdf

def gelu2(input_tensor):
  cdf = 0.5 * (1.0 + tf.math.erf(input_tensor / tf.sqrt(2.0)))
  return input_tensor * cdf


# +
x = np.linspace(-5, 5, 100)
y1 = gelu1(x)
y2 = gelu2(x)

plt.figure(figsize=(15, 10))
plt.plot(x, y1, '.')
plt.plot(x, y2)
plt.title("Gelu activation comparison")
plt.legend(["Gelu BioBERT", "Gelu Google BERT"]);

# +
x = np.linspace(-10, 10, 300)
y = (gelu1(x).numpy() - gelu2(x).numpy())

plt.figure(figsize=(15, 10))
plt.plot(x, y, '.-')
plt.title("Difference Gelu1 - Gelu2");

# +
x = np.linspace(-10, 10, 300)
y1 = np.diff(gelu1(x), prepend=[0])
y2 = np.diff(gelu2(x), prepend=[0])

plt.figure(figsize=(15, 10))
plt.plot(x, y1, '.-')
plt.plot(x, y2)
plt.title("Difference Gelu1 - Gelu2");

# +
x = np.linspace(-10, 10, 300)
y1 = np.diff(gelu1(x), prepend=[0, 0], n=2)
y2 = np.diff(gelu2(x), prepend=[0, 0], n=2)

plt.figure(figsize=(15, 10))
plt.plot(x, y1, '.-')
plt.plot(x, y2)
plt.title("Difference Gelu1 - Gelu2");
# -



StarcoderdataJupyterScriptsDedupFiltered