# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# # Gated Feedback RNN - Notes
#
# Paper's notation
# input is $x_t$
#
#
#
#
# Zaremba (2014) variant of LSTM
# $$\tilde{c_t} = tanh(W_c x_t + U_c h_{t-1})\,-\,new\,content$$
# $$c_t = f_tc_{t-1} + i_t \tilde{c_t}\,-\, content\,of\,memory\,cell$$
# $$i_t = \sigma(W_ix_t+U_ih_{t-1})\,-\,input\,gate$$
# $$f_t = \sigma(W_fx_t+U_fh_{t-1})\,-\,forget\,gate$$
# $$h_t = o_ttanh(c_t)\,-\,hidden \, state$$
# $$o_t = \sigma(W_ox_t + U_oh_{t-1})\,-\,output\,gate$$
#
# Modification for Gated Feedback LSTM
# $$\tilde{c_t^j} = tanh(W_c^{j-1\rightarrow j}h_t^{j-1} + \sum_{i=1}^L g^{i\rightarrow j}U_c^{i\rightarrow j}h_{t-1}^i)$$
#
#
# Gated Recurrent Unit, Cho (2014)
# $$h_t = (1-z_t)h_{t-1}+z_t \tilde{h_t}\,-\,GRU\, state$$
# $$z_t = \sigma(W_zx_t+U_zh_{t-1})\,-\, update\,gate$$
# $$\tilde{h_t} = tanh(Wx_t + r_tUh_{t-1})\,-\,new\,memory\,content$$
# $$r_t = \sigma(W_rx_t + U_rh_{t-1})\,-\,reset\,gate$$
#
# Modification for Gated Feedback GRU
# $$\tilde{h_t^j} = tanh(W^{j-1\rightarrow j}h_t^{j-1} + r_t^j \odot \sum_{i=1}^L g^{i\rightarrow j}U_c^{i\rightarrow j}h_{t-1}^i)$$
#
#
#
#
# ## Elman net
#
#
# $$h_t = W_hx_t+U_hh_{t-1}\,-\,hidden \, state$$
#
# 2-Layer Elman-RNN
# $$h_t^1 = W_h^1x_t+U_h^1h_{t-1}^1\,-\,hidden \, state\,1$$
# $$h_t^2 = W_h^2h_t^1+U_h^2h_{t-1}^2\,-\,hidden \, state\,2$$
#
# 2-Layer gated feedback RNN - GF-RNN = 2-layer Elman-RNN plus
# $$g^{1\rightarrow2} = \sigma(W_g^{1\rightarrow2}h_t^1 + U_g^{1\rightarrow2}\left[\begin{array}{c}
# h_{t-1}^1\\
# h_{t-1}^2\end{array}\right])$$
# $$g^{2\rightarrow1} = \sigma(W_g^{2\rightarrow1}x_t + U_g^{2\rightarrow1}\left[\begin{array}{c}
# h_{t-1}^1\\
# h_{t-1}^2\end{array}\right])$$
#
# Assuming that
# $$h_{t-1}^*=\left[\begin{array}{c}
# h_{t-1}^1\\
# h_{t-1}^2\end{array}\right]$$
#
# Layer-by-layer state - 
# $$h_t^j = tanh(W^{j-1\rightarrow j}h_t^{j-1} + \sum_{i=1}^L g^{i\rightarrow j}U^{i\rightarrow j}h_{t-1}^i)$$
#
# For 2-layer gated rf Elman - 
# $$h_t^1 = tanh(W^{0\rightarrow 1}x_t + g^{1\rightarrow 1}U^{1\rightarrow 1}h_{t-1}^1 + g^{2\rightarrow 1}U^{2\rightarrow 1}h_{t-1}^2)$$
# $$h_t^2 = tanh(W^{1\rightarrow 2}h_t^1 + g^{1\rightarrow 2}U^{1\rightarrow 2}h_{t-1}^1 + g^{2\rightarrow 2}U^{2\rightarrow 2}h_{t-1}^2)$$
#
#



StarcoderdataJupyterScriptsDedupFiltered