<jupyter_start><jupyter_text>Geron's Chapter 15: Modified for multivariate series (2 in, 2 out)n_steps != None anymore for cases also **TODO**: Still need a clear distinction between different classes of models that take in multivariate multi-step input Code borrows heavily from <NAME>'s 2nd edition: https://github.com/ageron/handson-ml2/blob/master/15_processing_sequences_using_rnns_and_cnns.ipynb  Setup First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20 and TensorFlow ≥2.0-preview.<jupyter_code># Python ≥3.5 is required
import sys
assert sys.version_info >= (3, 5)

# Scikit-Learn ≥0.20 is required
import sklearn
assert sklearn.__version__ >= "0.20"

# TensorFlow ≥2.0-preview is required
import tensorflow as tf
from tensorflow import keras
assert tf.__version__ >= "2.0"

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)
tf.random.set_seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "rnn"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID)
os.makedirs(IMAGES_PATH, exist_ok=True)

def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=300):
    path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)<jupyter_output><empty_output><jupyter_text>Generate the Dataset<jupyter_code>def generate_time_series(batch_size, n_steps):
    freq1, freq2, offsets1, offsets2 = np.random.rand(4, batch_size, 1)
    time = np.linspace(0, 1, n_steps)
    series = 0.5 * np.sin((time - offsets1) * (freq1 * 10 + 10))  #   wave 1
    series += 0.2 * np.sin((time - offsets2) * (freq2 * 20 + 20)) # + wave 2
    series += 0.1 * (np.random.rand(batch_size, n_steps) - 0.5)   # + noise
    return series[..., np.newaxis].astype(np.float32)<jupyter_output><empty_output><jupyter_text>TWO problems1. Random seed same leads to the same time series. Changed it.2. For arrays with dimensions > 2, hstack always adds arrays along the 2nd axis (or [1]). Had to use:```pythonnp.stack((array1, array2), axis=-1)```to make sure that the arrays were being stacked along the last axis.<jupyter_code>n_steps = 50

## SERIES 1
np.random.seed(42)
series = generate_time_series(10000, n_steps + 1)

## SERIES 2
np.random.seed(11)
series2 = generate_time_series(10000, n_steps + 1)

## Train, validation, test split
np.random.seed(42)
X_train, y_train = np.stack((series[:7000, :n_steps, -1],    series2[:7000, :n_steps, -1]),axis=-1),     np.hstack((series[:7000, -1],     series2[:7000, -1]))
X_valid, y_valid = np.stack((series[7000:9000, :n_steps,-1], series2[7000:9000, :n_steps, -1]),axis=-1), np.hstack((series[7000:9000, -1], series2[7000:9000, -1]))
X_test,  y_test  = np.stack((series[9000:, :n_steps, -1],    series2[9000:, :n_steps, -1]),axis=-1)   ,  np.hstack((series[9000:, -1],    series2[9000:, -1]))
print(X_train.shape, y_train.shape)
print(X_valid.shape, y_valid.shape)
print(X_test.shape, y_test.shape)
def plot_series(series, y=None, y_pred=None, x_label="$t$", y_label="$x(t)$"):
    plt.plot(series, ".-")
    if y is not None:
        plt.plot(n_steps, y, "bx", markersize=10)
    if y_pred is not None:
        plt.plot(n_steps, y_pred, "ro")
    plt.grid(True)
    if x_label:
        plt.xlabel(x_label, fontsize=16)
    if y_label:
        plt.ylabel(y_label, fontsize=16, rotation=0)
    plt.hlines(0, 0, 100, linewidth=1)
    plt.axis([0, n_steps + 1, -1, 1])

fig, axes = plt.subplots(nrows=1, ncols=3, sharey=True, figsize=(12, 4))
for col in range(3):
    plt.sca(axes[col])
    plot_series(X_valid[col, :, 0], y_valid[col, 0],
                y_label=("$x(t)$" if col==0 else None))
    plot_series(X_valid[col, :, 1], y_valid[col, 1],
                y_label=("$x(t)$" if col==0 else None))
save_fig("time_series_plot")
plt.show()<jupyter_output>Saving figure time_series_plot<jupyter_text>Computing Some Baselines Naive predictions (just predict the last observed value):<jupyter_code>y_pred  = X_valid[:, -1, 0]
y_pred2 = X_valid[:, -1, 1]

print(y_pred.shape, y_pred2.shape)

print(f"Series 1: {np.mean(keras.losses.mean_squared_error(y_valid[:,0], y_pred))}")
print(f"Series 2: {np.mean(keras.losses.mean_squared_error(y_valid[:,1], y_pred2))}")
plot_series(X_valid[0, :, 0], y_valid[0, 0], y_pred[0])
plot_series(X_valid[0, :, 1], y_valid[0, 1], y_pred2[0])
plt.show()<jupyter_output><empty_output><jupyter_text>Dense layer with one neuron = LINEAR regressionNow adapted for **multivariate regression** Linear predictions:<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)

# Multivariate shapes
n_input_features  = 2
n_output_features = 2

model = keras.models.Sequential([
    keras.layers.Flatten(input_shape=[50, n_input_features]),
    keras.layers.Dense(n_output_features)
])

model.compile(loss="mse", optimizer="adam")
history = model.fit(X_train, y_train, epochs=20,
                    validation_data=(X_valid, y_valid))
model.evaluate(X_valid, y_valid)<jupyter_output>2000/2000 [==============================] - 0s 18us/sample - loss: 0.0043<jupyter_text>Model summary and plot with shapes<jupyter_code>model.summary()
# !pip install pydot --- otherwise won't work
keras.utils.plot_model(model, 'dense_50.png', show_shapes=True)
def plot_learning_curves(loss, val_loss):
    plt.plot(np.arange(len(loss)) + 0.5, loss, "b.-", label="Training loss")
    plt.plot(np.arange(len(val_loss)) + 1, val_loss, "r.-", label="Validation loss")
    plt.gca().xaxis.set_major_locator(mpl.ticker.MaxNLocator(integer=True))
    plt.axis([1, 20, 0, 0.05])
    plt.legend(fontsize=14)
    plt.xlabel("Epochs")
    plt.ylabel("Loss")
    plt.grid(True)

plot_learning_curves(history.history["loss"], history.history["val_loss"])
plt.show()
y_pred = model.predict(X_valid)
plot_series(X_valid[0, :, 0], y_valid[0, 0], y_pred[0, 0])
plot_series(X_valid[0, :, 1], y_valid[0, 1], y_pred[0, 1])
plt.show()<jupyter_output><empty_output><jupyter_text>Using a Simple RNN<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)

# Multivariate shapes
n_input_features  = 2
n_output_features = 2

model = keras.models.Sequential([
    keras.layers.SimpleRNN(n_output_features, input_shape=[None, n_input_features])
])

optimizer = keras.optimizers.Adam(lr=0.005)
model.compile(loss="mse", optimizer=optimizer)
history = model.fit(X_train, y_train, epochs=10,
                    validation_data=(X_valid, y_valid))
model.evaluate(X_valid, y_valid)
plot_learning_curves(history.history["loss"], history.history["val_loss"])
plt.show()
y_pred = model.predict(X_valid)
plot_series(X_valid[0, :, 0], y_valid[0, 0], y_pred[0, 0])
plot_series(X_valid[0, :, 1], y_valid[0, 1], y_pred[0, 1])
plt.show()
# !pip install pydot --- otherwise won't work
keras.utils.plot_model(model, 'rnn_1.png', show_shapes=True)<jupyter_output><empty_output><jupyter_text>Deep RNNs<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)

# Multivariate shapes
n_input_features  = 2
n_output_features = 2

model = keras.models.Sequential([
    keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, n_input_features]),
    keras.layers.SimpleRNN(20),
    keras.layers.Dense(n_output_features)
])

model.compile(loss="mse", optimizer="adam")
history = model.fit(X_train, y_train, epochs=20,
                    validation_data=(X_valid, y_valid))
model.evaluate(X_valid, y_valid)
plot_learning_curves(history.history["loss"], history.history["val_loss"])
plt.show()
y_pred = model.predict(X_valid)
plot_series(X_valid[0, :, 0], y_valid[0, 0], y_pred[0, 0])
plot_series(X_valid[0, :, 1], y_valid[0, 1], y_pred[0, 1])
plt.show()
# !pip install pydot --- otherwise won't work
keras.utils.plot_model(model, 'rnn_2_dense_end.png', show_shapes=True)<jupyter_output><empty_output><jupyter_text>MULTI-STEP forecasting  Method 1: 1. Take models from before.2. Run a loop over 10 points to get prediction for each point<jupyter_code>np.random.seed(43) # not 42, as it would give the first series in the train set
series = generate_time_series(1, n_steps + 10)

np.random.seed(12)
series2 = generate_time_series(1, n_steps + 10)

X_new,  Y_new  = np.stack((series[:, :n_steps, -1], series2[:, :n_steps, -1]),axis=-1)   , np.stack((series[:, n_steps:, -1], series2[:, n_steps:, -1]), axis=-1)
X = X_new

for step_ahead in range(10):
    y_pred_one = model.predict(X[:, step_ahead:, :])[:, np.newaxis, :]
    print(y_pred_one.shape)
    X = np.concatenate([X, y_pred_one], axis=1)

Y_pred = X[:, n_steps:]
Y_pred.shape
def plot_multiple_forecasts(X, Y, Y_pred):
    n_steps = X.shape[1]
    ahead = Y.shape[1]
    plot_series(X[0, :, 0])
    plt.plot(np.arange(n_steps, n_steps + ahead), Y_pred[0, :, 0], "ro-", label="Actual")
    plt.plot(np.arange(n_steps, n_steps + ahead), Y[0, :, 0], "bx-", label="Forecast", markersize=10)
    plt.axis([0, n_steps + ahead, -1, 1])
    plt.legend(fontsize=14)

plot_multiple_forecasts(X_new, Y_new, Y_pred)
save_fig("forecast_ahead_plot")
plt.show()
def plot_multiple_forecasts2(X, Y, Y_pred):
    n_steps = X.shape[1]
    ahead = Y.shape[1]
    plot_series(X[0, :, 1])
    plt.plot(np.arange(n_steps, n_steps + ahead), Y_pred[0, :, 1], "ro-", label="Actual")
    plt.plot(np.arange(n_steps, n_steps + ahead), Y[0, :, 1], "bx-", label="Forecast", markersize=10)
    plt.axis([0, n_steps + ahead, -1, 1])
    plt.legend(fontsize=14)
    
    
plot_multiple_forecasts2(X_new, Y_new, Y_pred)
save_fig("forecast_ahead_plot2")
plt.show()<jupyter_output>Saving figure forecast_ahead_plot2<jupyter_text>Method 21. Predict 10 steps at ONCE.2. Have to train a NEW model now, cannot use previous one.This took the longest time because had to use `np.stack` instead of np.hstack for y_train, y_valid, y_test also since there are 3 axis dimensions there!!Also, forgot to change the neuron dimension inside Dense to n_output_features from 1. Now let's create an RNN that predicts all 10 next values at once:<jupyter_code>n_steps = 50

## SERIES 1
np.random.seed(42)
series = generate_time_series(10000, n_steps + 10)

## SERIES 2
np.random.seed(11)
series2 = generate_time_series(10000, n_steps + 10)

## Train, validation, test split
# X's remain the same... notice the change in y-indices!!

np.random.seed(42)
X_train, y_train = np.stack((series[:7000, :n_steps, -1],    series2[:7000, :n_steps, -1]),axis=-1),     np.stack((series[:7000, -10:, -1],    series2[:7000, -10:, -1]), axis=-1)
X_valid, y_valid = np.stack((series[7000:9000, :n_steps,-1], series2[7000:9000, :n_steps, -1]),axis=-1), np.stack((series[7000:9000, -10:, -1],series2[7000:9000, -10:, -1]), axis=-1)
X_test,  y_test  = np.stack((series[9000:, :n_steps, -1],    series2[9000:, :n_steps, -1]),axis=-1)   ,  np.stack((series[9000:, -10:, -1],    series2[9000:, -10:, -1]), axis=-1)
print(X_train.shape, y_train.shape)
print(X_valid.shape, y_valid.shape)
print(X_test.shape, y_test.shape)<jupyter_output>(7000, 50, 2) (7000, 10, 2)
(2000, 50, 2) (2000, 10, 2)
(1000, 50, 2) (1000, 10, 2)<jupyter_text>This was giving a whole lot of issuesGeron's earlier version had different layers that are commented out now.His new version was difficult to fit with dimensions. Finally had to introduce n_steps in input_shape in the first layers.<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)

n_output = y_valid.shape[1] * y_valid.shape[2]

y_train = y_train.reshape((y_train.shape[0], n_output))
y_valid = y_valid.reshape((y_valid.shape[0], n_output))

print(y_train.shape, y_valid.shape)
print(X_train.shape)

model = keras.models.Sequential([
    keras.layers.SimpleRNN(32, recurrent_dropout=0.2, return_sequences=True, input_shape=(n_steps, n_input_features)), # shouldn't None = n_steps????? That's what I understand from Brownlee and Chollet's book
    keras.layers.SimpleRNN(32, recurrent_dropout=0.2, dropout=0.2),#, return_sequences=True),
    #keras.layers.TimeDistributed(keras.layers.Dense(n_output_features)),
    #keras.layers.Lambda(lambda y_pred: y_pred[:, -10:, :]) # added a column for feature axes
    #keras.layers.Dense(32),
    #keras.layers.BatchNormalization(),
    keras.layers.Dense(n_output)
])

model.compile(loss="mse", optimizer="adam")
history = model.fit(X_train, y_train, epochs=10, validation_data=(X_valid, y_valid))
np.random.seed(43) # not 42, as it would give the first series in the train set
series = generate_time_series(1, 50 + 10)

np.random.seed(12)
series2 = generate_time_series(1, 50 + 10)

X_new, Y_new  = np.stack((series[:, :50, -1], series2[:, :50, -1]),axis=-1), np.stack((series[:, -10:, -1], series2[:, -10:, -1]), axis=-1)
X = X_new

Y_pred = model.predict(X_new)#[:, :]
Y_pred.shape
def plot_multiple_forecasts(X, Y, Y_pred):
    n_steps = X.shape[1]
    ahead = Y.shape[1]
    plot_series(X[0, :, 0])
    plt.plot(np.arange(n_steps, n_steps + ahead), Y_pred[0, :10], "ro-", label="Actual")
    plt.plot(np.arange(n_steps, n_steps + ahead), Y[0, :, 0], "bx-", label="Forecast", markersize=10)
    plt.axis([0, n_steps + ahead, -1, 1])
    plt.legend(fontsize=14)

plot_multiple_forecasts(X_new, Y_new, Y_pred)
plt.show()
def plot_multiple_forecasts2(X, Y, Y_pred):
    n_steps = X.shape[1]
    ahead = Y.shape[1]
    plot_series(X[0, :, 1])
    plt.plot(np.arange(n_steps, n_steps + ahead), Y_pred[0, 10:], "ro-", label="Actual")
    plt.plot(np.arange(n_steps, n_steps + ahead), Y[0, :, 1], "bx-", label="Forecast", markersize=10)
    plt.axis([0, n_steps + ahead, -1, 1])
    plt.legend(fontsize=14)

plot_multiple_forecasts2(X_new, Y_new, Y_pred)
plt.show()<jupyter_output><empty_output><jupyter_text>New metrics<jupyter_code>def last_10_time_steps_mse(Y_true, Y_pred):
    return keras.metrics.mean_squared_error(Y_true[:, -10:], Y_pred[:, -10:])
def last_5_time_steps_mse(Y_true, Y_pred):
    return keras.metrics.mean_squared_error(Y_true[:, -5:], Y_pred[:, -5:])<jupyter_output><empty_output><jupyter_text>Method 3Goal: Forecast 10:59 based on 0:49.Previously, 50:59 was being predicted by 0:49. Now let's create an RNN that predicts the input sequence, shifted 10 steps into the future. That is, instead of just forecasting time steps 50 to 59 based on time steps 0 to 49, it will forecast time steps 10 to 59 based on time steps 0 to 49 (the time steps 10 to 49 are in the input, but the model is causal so at any time step it cannot see the future inputs):<jupyter_code>n_steps = 50

## SERIES 1
np.random.seed(42)
series = generate_time_series(10000, n_steps + 10)

## SERIES 2
np.random.seed(11)
series2 = generate_time_series(10000, n_steps + 10)

## Train, validation, test split
# X's remain the same... notice the change in y-indices!!

np.random.seed(42)
X_train, y_train = np.stack((series[:7000, :n_steps, -1],    series2[:7000, :n_steps, -1]),axis=-1),     np.stack((series[:7000,  10:, -1],    series2[:7000,  10:, -1]), axis=-1)
X_valid, y_valid = np.stack((series[7000:9000, :n_steps,-1], series2[7000:9000, :n_steps, -1]),axis=-1), np.stack((series[7000:9000,  10:, -1],series2[7000:9000,  10:, -1]), axis=-1)
X_test,  y_test  = np.stack((series[9000:, :n_steps, -1],    series2[9000:, :n_steps, -1]),axis=-1)   ,  np.stack((series[9000:,  10:, -1],    series2[9000:,  10:, -1]), axis=-1)
#np.random.seed(42)

#n_steps = 50
#series = generate_time_series(10000, n_steps + 10)
#X_train, Y_train = series[:7000, :n_steps], series[:7000, 10:]
#X_valid, Y_valid = series[7000:9000, :n_steps], series[7000:9000, 10:]
#X_test, Y_test = series[9000:, :n_steps], series[9000:, 10:]
X_train.shape, y_train.shape
np.random.seed(42)
tf.random.set_seed(42)

model = keras.models.Sequential([
    keras.layers.SimpleRNN(20, return_sequences=True, input_shape=[None, n_input_features]),
    keras.layers.SimpleRNN(20, recurrent_dropout=0.2, return_sequences=True),
    keras.layers.TimeDistributed(keras.layers.Dense(n_output_features))
])

model.compile(loss="mse", optimizer="adam", metrics=[last_10_time_steps_mse])
history = model.fit(X_train, y_train, epochs=20,
                    validation_data=(X_valid, y_valid))
np.random.seed(43) # not 42, as it would give the first series in the train set
series = generate_time_series(1, 50 + 10)

np.random.seed(12)
series2 = generate_time_series(1, 50 + 10)

X_new, Y_new  = np.stack((series[:, :50, -1], series2[:, :50, -1]),axis=-1),      np.stack((series[:, 50:, -1], series2[:, 50:, -1]), axis=-1)
X = X_new

Y_pred = model.predict(X_new)[:, -10:, :]
def plot_multiple_forecasts(X, Y, Y_pred):
    n_steps = X.shape[1]
    ahead = Y.shape[1]
    plot_series(X[0, :, 0])
    plt.plot(np.arange(n_steps, n_steps + ahead), Y_pred[0, :, 0], "ro-", label="Actual")
    plt.plot(np.arange(n_steps, n_steps + ahead), Y[0, :, 0], "bx-", label="Forecast", markersize=10)
    plt.axis([0, n_steps + ahead, -1, 1])
    plt.legend(fontsize=14)

plot_multiple_forecasts(X_new, Y_new, Y_pred)
save_fig("forecast_ahead_plot")
plt.show()
def plot_multiple_forecasts2(X, Y, Y_pred):
    n_steps = X.shape[1]
    ahead = Y.shape[1]
    plot_series(X[0, :, 1])
    plt.plot(np.arange(n_steps, n_steps + ahead), Y_pred[0, :, 1], "ro-", label="Actual")
    plt.plot(np.arange(n_steps, n_steps + ahead), Y[0, :, 1], "bx-", label="Forecast", markersize=10)
    plt.axis([0, n_steps + ahead, -1, 1])
    plt.legend(fontsize=14)
    
    
plot_multiple_forecasts2(X_new, Y_new, Y_pred)
save_fig("forecast_ahead_plot2")
plt.show()<jupyter_output>Saving figure forecast_ahead_plot2<jupyter_text>LSTMs<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)

model = keras.models.Sequential([
    keras.layers.LSTM(20, return_sequences=True, input_shape=[None, n_input_features]),
    keras.layers.LSTM(20, return_sequences=True),
    keras.layers.TimeDistributed(keras.layers.Dense(n_output_features))
])

model.compile(loss="mse", optimizer="adam", metrics=[last_10_time_steps_mse])
history = model.fit(X_train, y_train, epochs=20,
                    validation_data=(X_valid, y_valid))
model.evaluate(X_valid, y_valid)
plot_learning_curves(history.history["loss"], history.history["val_loss"])
plt.show()
np.random.seed(43) # not 42, as it would give the first series in the train set
series = generate_time_series(1, 50 + 10)

np.random.seed(12)
series2 = generate_time_series(1, 50 + 10)

X_new, Y_new  = np.stack((series[:, :50, -1], series2[:, :50, -1]),axis=-1),      np.stack((series[:, 50:, -1], series2[:, 50:, -1]), axis=-1)
X = X_new

Y_pred = model.predict(X_new)[:, -10:, :]
plot_multiple_forecasts(X_new, Y_new, Y_pred)
plt.show()
plot_multiple_forecasts2(X_new, Y_new, Y_pred)
plt.show()<jupyter_output><empty_output><jupyter_text>Using One-Dimensional Convolutional Layers to Process Sequences ```1D conv layer with kernel size 4, stride 2, VALID padding:              |-----2----|      |-----5---...         |----23-----|        |-----1----|      |-----4-----|   ...   |-----22----|  |-----0----|      |-----3----|      |---...-21---|X: 0  1  2  3  4  5  6  7  8  9  10 11 12 ... 43 44 45 46 47 48 49Y: 10 11 12 13 14 15 16 17 18 19 20 21 22 ... 53 54 55 56 57 58 59Output:X: 0  1  2  3  4  5  ... 19 20 21 22 23Y: 13 15 17 19 21 23 ... 51 53 55 57 59```<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)

model = keras.models.Sequential([
    keras.layers.Conv1D(filters=20, kernel_size=4, strides=2, padding="VALID",
                        input_shape=[None, n_input_features]),
    keras.layers.LSTM(20, return_sequences=True),
    keras.layers.LSTM(20, return_sequences=True),
    keras.layers.TimeDistributed(keras.layers.Dense(n_output_features))
])

model.compile(loss="mse", optimizer="adam", metrics=[last_5_time_steps_mse])
history = model.fit(X_train, y_train[:, 3::2], epochs=20,
                    validation_data=(X_valid, y_valid[:, 3::2]))
model.evaluate(X_valid, y_valid[:, 3::2])
plot_learning_curves(history.history["loss"], history.history["val_loss"])
plt.show()
model.summary()
keras.utils.plot_model(model, 'cnn.png', show_shapes=True)<jupyter_output><empty_output><jupyter_text>WaveNet ```C2     /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\.../\ /\ /\ /\ /\ /\       /  \  /  \  /  \  /  \  /  \  /  \       /  \  /  \  /  \     /    \      /    \      /    \                 /    \C1  /\ /\ /\ /\ /\ /\ /\ /\ /\ /\ /\  /\ /.../\ /\ /\ /\ /\ /\ /\X: 0  1  2  3  4  5  6  7  8  9  10 11 12 ... 43 44 45 46 47 48 49Y: 10 11 12 13 14 15 16 17 18 19 20 21 22 ... 53 54 55 56 57 58 59Output:X: 0  1  2  3  4  5  ... 19 20 21 22 23Y: 13 15 17 19 21 23 ... 51 53 55 57 59```  Conv1D output has filters as last dimension so it has to be changed to n_output_features<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)

model = keras.models.Sequential()
model.add(keras.layers.InputLayer(input_shape=[None, n_input_features]))
for rate in (1, 2, 4, 8) * 2:
    #model.add(keras.layers.Lambda(
    #    lambda inputs: keras.backend.temporal_padding(inputs, (rate, 0))))
    model.add(keras.layers.Conv1D(filters=20, kernel_size=2, padding="causal",
                                  activation="relu", dilation_rate=rate))
model.add(keras.layers.Conv1D(filters=n_output_features, kernel_size=1))
model.compile(loss="mse", optimizer="adam", metrics=[last_10_time_steps_mse])
history = model.fit(X_train, y_train, epochs=20,
                    validation_data=(X_valid, y_valid))
model.evaluate(X_valid, y_valid)
plot_learning_curves(history.history["loss"], history.history["val_loss"])
plt.show()
model.summary()
keras.utils.plot_model(model, 'cnn_wavenetsimple_causal.png', show_shapes=True)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup