# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# + [markdown] id="gW-Ju3o3WxJj"
# ### Zaimportowanie bibliotek
# -

import datetime
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt

print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))

# + [markdown] id="xM12BryyW1Uz"
# ### Wczytanie wierszy i odpowiednich kolumn z plików 

# + id="xok-RpbxznTO"
columns = ['station_id', 'day', 'month', 'year', 'mean temperature [deg F]', 'mean dew point [deg F]', 'mean pressure (sea level) [Pa]', 'mean pressure (station) [Pa]', 'mean visibility [mile]', 'mean wind speed [knot]', 'max wind gust [knot]', 'max temperature [deg F]',
'min temperature [deg F]', 'total precipitation [inch]', 'snow depth [inch]']
train_df = pd.read_excel('data_JA_df_training.xlsx')[columns]
test_df = pd.read_excel('data_JA_df_evaluation.xlsx')[columns]

# + [markdown] id="-Hli_B2JX2sn"
# ### Znormalizowanie danych

# + id="XriI-lSqX1sg"
columns = ['station_id', 'mean dew point [deg F]', 'mean pressure (sea level) [Pa]', 'mean pressure (station) [Pa]', 'mean visibility [mile]', 'mean wind speed [knot]', 'max wind gust [knot]', 'max temperature [deg F]',
'min temperature [deg F]', 'total precipitation [inch]', 'snow depth [inch]']

train_mean = train_df[columns].mean()
train_std = train_df[columns].std()

test_mean = test_df[columns].mean()
test_std = test_df[columns].std()

train_df[columns] = (train_df[columns] - train_mean) / train_std
test_df[columns] = (test_df[columns] - test_mean) / test_std

# + [markdown] id="YoFS71TnW8MH"
# ### Dodanie kolumny z datą do późniejszego przygotowania sekwencji

# + id="ioit4mGHTpgM"
train_df['date'] = train_df.apply(lambda d: datetime.date(day=int(d.day), month=int(d.month), year=int(d.year)), axis=1)
test_df['date'] = test_df.apply(lambda d: datetime.date(day=int(d.day), month=int(d.month), year=int(d.year)), axis=1)

# + [markdown] id="KcAicOCPXBc5"
# ### Pogrupowanie po stacjach

# + colab={"base_uri": "https://localhost:8080/"} id="iYUEA976Mq28" outputId="31c23e36-575a-43ca-cf16-71b2678391b1"
df_stations_train = train_df.groupby("station_id")
df_stations_test = test_df.groupby("station_id")

print(df_stations_train.size())
print(df_stations_test.size())

# + [markdown] id="BnULcSQnXENY"
# ### Utworzenie sekwencji treningowej i testowej

# + id="JWYbtrSJPKOC"
two_weeks_diff = datetime.timedelta(days = 13) # w sumie 14 dni
st_index = 0
stations_train_value = []
stations_train_label = []
for name, station_train in df_stations_train:
  stations_train_value.append([])
  stations_train_label.append([])
  for index, station_measurement in station_train.iterrows():

    m_station_id = station_measurement.station_id
    m_date_to = station_measurement.date
    m_date_from = m_date_to - two_weeks_diff;

    two_weeks_sequence = station_train[(station_train['date'] >= m_date_from) & (station_train['date'] <= m_date_to)]
    if len(two_weeks_sequence) == 14:
      stations_train_value[st_index].append(two_weeks_sequence[columns])
      stations_train_label[st_index].append(station_measurement['mean temperature [deg F]'])
  st_index += 1

# + id="qdEDOwDpvPkZ"
two_weeks_diff = datetime.timedelta(days = 13) # w sumie 14 dni
st_index = 0
stations_test_value = []
stations_test_label = []
for name, station_train in df_stations_test:
  stations_test_value.append([])
  stations_test_label.append([])
  for index, station_measurement in station_train.iterrows():

    m_station_id = station_measurement.station_id
    m_date_to = station_measurement.date
    m_date_from = m_date_to - two_weeks_diff;

    two_weeks_sequence = station_train[(station_train['date'] >= m_date_from) & (station_train['date'] <= m_date_to)]
    if len(two_weeks_sequence) == 14:
      stations_test_value[st_index].append(two_weeks_sequence[columns])
      stations_test_label[st_index].append(station_measurement['mean temperature [deg F]'])
  st_index += 1

# + [markdown] id="kKT0kmi7pMfH"
# ### Zamiana list na numpy array

# + colab={"base_uri": "https://localhost:8080/"} id="y0hFUr4IO3Mn" outputId="af11239f-ad6e-4b9d-e0d4-ef7c530956aa"
train_final_value = np.array(stations_train_value)
train_final_label = np.array(stations_train_label)

print(train_final_value.shape)
print(train_final_label.shape)

for index, row in enumerate(train_final_value):
  train_final_value[index] = np.array(row).astype(np.float32)

for index, row in enumerate(train_final_label):
  train_final_label[index] = np.array(row).astype(np.float32)

print(train_final_value[0][0])
print(train_final_label[0][0])

# + colab={"base_uri": "https://localhost:8080/"} id="MZBBetSgvo5Z" outputId="e20e2d0f-b495-4191-ef0a-0d29f9a3ec44"
test_final_value = np.array(stations_test_value)
test_final_label = np.array(stations_test_label)

print(test_final_value.shape)
print(test_final_label.shape)

for index, row in enumerate(test_final_value):
  test_final_value[index] = np.array(row).astype(np.float32)

for index, row in enumerate(test_final_label):
  test_final_label[index] = np.array(row).astype(np.float32)

print(test_final_value[0][0])
print(test_final_label[0][0])

# + [markdown] id="ABd2G_Z6p_be"
# ### Połączenie sekwencji z różnych stacji

# + id="zCPS-qN-qILS"
train_final_value_concat = np.concatenate(train_final_value)
train_final_label_concat = np.concatenate(train_final_label)
test_final_value_concat = np.concatenate(test_final_value)
test_final_label_concat = np.concatenate(test_final_label)

# + [markdown] id="bXPnsadkutqL"
# ### Sprawdzenie przykładowych elementów

# + colab={"base_uri": "https://localhost:8080/"} id="z2A2Y1W1uw8g" outputId="a5c0161e-4631-4176-b27c-10d0ddec8a91"
print(train_final_value_concat[0])
print(train_final_label_concat[0])

# + colab={"base_uri": "https://localhost:8080/"} id="M_MPZI7WwARv" outputId="cbb949bb-1a59-4d5e-9d14-542639143285"
print(test_final_value_concat[0])
print(test_final_label_concat[0])

# + [markdown] id="DQ6ocStLpRk0"
# ### Zbudowanie modelu

# + colab={"base_uri": "https://localhost:8080/"} id="VHVRLZ0QyWPe" outputId="ae7bb6c9-46fb-4cc6-d551-8a91e1aca00e"
lstm_model_lr = tf.keras.models.Sequential([
    tf.keras.layers.LSTM(256, activation='relu', return_sequences=True),
    tf.keras.layers.LSTM(128, activation='relu', return_sequences=True),
    tf.keras.layers.LSTM(64, activation='relu'),
    tf.keras.layers.Dense(units=1, activation='relu')
])

# + colab={"base_uri": "https://localhost:8080/"} id="capXMiWgus7W" outputId="724b16ac-5995-4f4f-b6ae-7159abab619a"
MAX_EPOCHS = 30

early_stopping=tf.keras.callbacks.EarlyStopping(
    monitor='val_mean_squared_error', min_delta=0, patience=10, verbose=2, mode='auto',
    baseline=None, restore_best_weights=True)

lstm_model_lr.compile(loss=tf.losses.MeanSquaredError(),
              optimizer=tf.optimizers.Adam(learning_rate=0.0001),
              metrics=[tf.metrics.MeanSquaredError()],
)

history = lstm_model_lr.fit(train_final_value_concat, train_final_label_concat, validation_split=0.2, epochs=MAX_EPOCHS, callbacks=[early_stopping])

# + colab={"base_uri": "https://localhost:8080/", "height": 295} id="f_Hys0k-rTSX" outputId="42853e7e-352d-4673-9b32-3c74db37ca73"
plt.plot(history.history['mean_squared_error'])
plt.plot(history.history['val_mean_squared_error'])
plt.title('model MSE')
plt.ylabel('MSE')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

# + [markdown] id="s7Aj-o_swM6t"
# ### Ewaluacja

# + colab={"base_uri": "https://localhost:8080/"} id="3Zo0QkCsylFt" outputId="f2e95342-f4d3-4ce8-e184-4a86da87a6db"
lstm_model_lr.evaluate(test_final_value_concat, test_final_label_concat)

# + [markdown] id="7MN91LtB_wq9"
# ### Wyświetlenie wykresu

# + id="4Xbi2Srl_yOV"
predictions = lstm_model_lr.predict(test_final_value_concat, batch_size = 1)

start = 0
end = 200

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(test_final_label_concat[start:end], 'b', predictions[start:end], 'g')

StarcoderdataJupyterScriptsDedupFiltered