# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Django Shell-Plus
#     language: python
#     name: django_extensions
# ---

# +
from stations.models import Measurement, Prediction, Station
from asgiref.sync import sync_to_async
import pandas as pd
import matplotlib
import os
os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"


import pandas as pd
from datetime import datetime
from matplotlib import pyplot
from math import sqrt
from numpy import concatenate
from matplotlib import pyplot as plt
#import seaborn as sns
from pandas import read_csv
from pandas import DataFrame
from pandas import concat
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import mean_squared_error,mean_absolute_error
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from keras.callbacks import ModelCheckpoint,EarlyStopping
from keras.models import load_model, model_from_json
import pickle
import glob

from geolomasexp.configuration import LSTMTrainingScriptConfig

from geolomasexp.data import read_time_series_from_csv
from geolomasexp.feature import (
    get_dataset_from_series,
    get_interest_variable
)
from geolomasexp.model_hyperopt import (
    get_lstm_nnet_opt
)
from geolomasexp.model import (train_val_test_split)

from geolomasexp.model import predict_with_model


import ipdb


# -

def read_time_series_from_db(
            sensor='A620',
            date_col='date',
            hr_col='hr',
            min_col='minute',
            numeric_var='temperature',
            sensor_var='inme',  # TODO : change this for station_code in all the script
            date_since=None,
            which_minutes = [0,15,30,45]
            ):
        # get the station (sensor)
        #station = Station.objects.get(code=sensor)
        # get all the measurements from this station
        measurements = Measurement.objects.filter(station=Station.objects.get(
            code=sensor).id)
        if date_since is not None:
            measurements = measurements.filter(datetime__gte=date_since)

        print(
            'Measurements to read from database for sensor {} with query \n{}'.
            format(sensor, measurements.query))
        # get a dataframe from the measurements
        dataset = pd.DataFrame(
            list(measurements.values('datetime', 'attributes')))
        print('Dataset from database of shape {}'.format(
            dataset.shape))
        # parse datetime column to get sepearae date, hr and minute columns
        dataset[date_col] = dataset.datetime.dt.date
        dataset[hr_col] = dataset.datetime.dt.hour
        dataset[min_col] = dataset.datetime.dt.minute
        # get the numeric var column parsing the json
        dataset[numeric_var] = dataset.attributes.apply(
            lambda x: x[numeric_var])
        dataset[sensor_var] = sensor

        # sort and re-index before returning
        dataset.sort_values([date_col, hr_col,min_col], inplace=True, ascending=True)
        dataset.reset_index(inplace=True)

        return dataset.loc[dataset.minute.isin(which_minutes)]


dataframe_temp = read_time_series_from_db(which_minutes=[0])
dataframe_temp

from geolomasexp.feature import get_dataset_from_series, get_interest_variable
time_series_dset = get_interest_variable(dataframe_temp, 'inme',
                                                 'date', 'hr','temperature',
                                                 'A620')
time_series_dset.head()

# +
from geolomasexp.feature import series_to_supervised

#model_package_name = '../../satlomas-exp/models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-22_22:26:47.model'
#model_package_name = '../models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-22_23:09:03.model'
model_package_name = '../models/esp:100_eps:1000_loss:mean_absolute_error_opt:adam_pstps:24_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-28_02:57:24.model'

with open(model_package_name, 'rb') as file_pi:
    model_package = pickle.load(file_pi)
    
scaler = model_package['scaler']


# obtenemos los valores como una matriz
values = time_series_dset.values
# ensure all data is float
values = values.astype('float32')

scaled = scaler.transform(values)
scaled = pd.DataFrame(scaled,columns=time_series_dset.columns)# frame as supervised learning

reframed = series_to_supervised(scaled, 24, 1)



# +
from geolomasexp.model import (build_lstm_nnet, eval_regression_performance,
                               fit_model, train_val_test_split)

dataset_splits = train_val_test_split(reframed, 24,1, 'temperature')
dataset_splits

# +
dset = dataset_splits['trainset']

X = dset['X']
y = dset['y']

X[0,:].shape

y[0:24].reshape(1,24)
scaler.inverse_transform(y[0:24].reshape(1,24))

# +
from sklearn.metrics import mean_absolute_error , max_error
import numpy as np

dset = dataset_splits['trainset']

X = dset['X']
y = dset['y']

#y = y.reshape((len(y), 1))
#yhat = model.predict(X)

future_steps = 24

maes = []
maxes = []

#model_package_name = '../../satlomas-exp/models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-22_22:26:47.model'
#model_package_name = '../models/esp:100_eps:1000_loss:mean_absolute_error_opt:adam_pstps:24_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-28_02:57:24.model'


#with open(model_package_name, 'rb') as file_pi:
#    model_package = pickle.load(file_pi)
    
#scaler = model_package['scaler']



#for i in range(X.shape[0]-future_steps):
for i in np.random.randint(0,X.shape[0]-future_steps-1,10):    
    try:
        # do we need to add the first dimension 
        datapoint = X[i,:,:]
        print('datapoint {}'.format(datapoint))
        
        true = scaler.inverse_transform(y[i:i+future_steps].reshape(1,future_steps))
        print('true {}'.format(true))
        
        preds,_ = predict_with_model(datapoint,model_package_name,future_steps)
        pred_list = []
        for step in preds:
            pred_list.append(np.float64(step[0][0]))
        pred = np.array([pred_list])
        print('pred {}'.format(pred))
        
        error = pred - true
        print('error {}'.format(error))
        
        plt.figure()
        plt.plot(range(true.shape[1]),true[0],'g')
        plt.plot(range(pred.shape[1]),pred[0],'b')
        plt.show()
        
        mae_ = mean_absolute_error(true[0],pred[0])
        maes.append(mae_)
        print('mae {}'.format(mae_))
        maxe_ = max_error(true[0],pred[0])
        maxes.append(maxe_)
        print('maxe {}'.format(maxe_))
        
        max_error_index = np.argmax(error[0])
        print('index of max error {}'.format(max_error_index))
    except Exception as e:
        print('Exception {}'.format(e))


# -

print(np.mean(maes))
print(np.mean(maxes))


# +
def get_database_data(djmodel = Prediction,sensor = 'A620',numeric_vars = ['temperature'],last_n_steps = -1):
    stat_id = Station.objects.get(code=sensor).id
    if last_n_steps > 0:
        records = djmodel.objects.filter(station=stat_id).order_by('-datetime')[:last_n_steps]
    else :
        records = djmodel.objects.filter(station=stat_id)
    dataset = pd.DataFrame(list(records.values('datetime', 'attributes')))
    dataset['date'] = dataset.datetime.dt.date
    dataset['hr'] = dataset.datetime.dt.hour
    dataset['minute'] = dataset.datetime.dt.minute
    # get the numeric var column parsing the json
    for numeric_var in numeric_vars:
        dataset[numeric_var] = dataset.attributes.apply(lambda x: x[numeric_var])
    # sort and re-index before returning
    dataset.sort_values(['date', 'hr','minute'], inplace=True, ascending=True)
    dataset.reset_index(inplace=True)
    return dataset

predictions_df = get_database_data(numeric_vars = ['temperature','humidity'])
predictions_df.sort_values('datetime',ascending = True,inplace = True)
predictions_df.set_index('datetime',inplace=True)
predictions_df
# -

measurement_df = get_database_data(Measurement,numeric_vars = ['temperature','humidity'],last_n_steps=30)
measurement_df.sort_values('datetime',ascending = True,inplace = True)
measurement_df.set_index('datetime',inplace=True)
measurement_df

plot_df = pd.concat([predictions_df[['temperature']],measurement_df[['temperature']]])
plot_df.sort_values('datetime',ascending = True,inplace= True)

from matplotlib import pyplot as plt
plt.figure(figsize = (15,10))
plot_df.temperature.plot()

from matplotlib import pyplot as plt
plt.figure(figsize = (15,10))
measurement_df.temperature.plot()
predictions_df.temperature[:30].plot()
plt.show()

from matplotlib import pyplot as plt
plt.figure(figsize = (15,10))
measurement_df.humidity.plot()
predictions_df.humidity[:30].plot()
plt.show()

from matplotlib import pyplot as plt
plt.figure(figsize = (15,10))
measurement_df.sort_values('datetime',ascending = True).set_index('datetime').temperature.plot()
predictions_df.sort_values('datetime',ascending = True).set_index('datetime').temperature[:20].plot()
plt.show()

# +

out_model_name = glob.glob('../models/*_hyperopt_model_*.hdf5')[-1]
out_model_name = '../models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_hyperopt_model_2020-04-20_23:16:42.hdf5'
print(out_model_name)
#out_model_name = '../models/esp:10_eps:2_loss:mean_squared_error_opt:adam_pstps:3_sensor:A620_var:temp_basenet:4.4_midnet:4.2_hyperoptpars:[1, 2, 3][2, 3][0.1, 0.8]3_hyperopt_model_2020-02-25_10:13:29.hdf5'
model_hyper = load_model(out_model_name)
print(model_hyper.summary())


# +
model_pckg_filename = '../models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-20_23:16:42.model'
with open(model_pckg_filename, 'rb') as file_pi:
    model_package = pickle.load(file_pi)
    
model = model_package['model']
print(model.summary())
# -

import numpy as np
scaler = model_package['scaler']
datapoint = measurement_df.temperature.values[:8]
print(datapoint)
for step in range(10) :
    X = np.array(datapoint[-8:]).reshape(1,model.input_shape[1],model.input_shape[2])
    y = scaler.inverse_transform(model.predict(X))
    datapoint = np.append(datapoint,y)
print(datapoint)

np.array(datapoint[-8:]).reshape(1,model.input_shape[1],model.input_shape[2]).shape

measurement_df.temperature.values[:8+10]

data_csv = pd.read_csv('../../satlomas-exp/data/meteorologico/A620_temp.csv')
data_csv

from geolomasexp.model import predict_with_model
start_point = -1 * 24 * 10
steps_train = 8
datapoint = data_csv.temperature.values[start_point:start_point+steps_train]
datapoint

model_package_name = '../../satlomas-exp/models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-22_22:26:47.model'
#model_package_name = '../models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-22_23:09:03.model'
pred,_ = predict_with_model(datapoint,model_package_name,24)
pred

data_csv.temperature.values[start_point+steps_train-1:start_point+steps_train+24]

datapoint_db = dataframe_temp.temperature.values[start_point:start_point+steps_train]
datapoint_db

model_package_name = '../../satlomas-exp/models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-22_22:26:47.model'
#model_package_name = '../models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-22_23:09:03.model'
pred,_ = predict_with_model(datapoint_db,model_package_name,24)
pred

import numpy as np
datapoint_db.shape
X = np.array(datapoint_db).reshape(1,8,1)
X

# +
model_package_name = '../../satlomas-exp/models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-22_22:26:47.model'
#model_package_name = '../models/esp:10_eps:200_loss:mean_squared_error_opt:adam_pstps:8_sensor:A620_var:temperature_basenet:4.4_midnet:4.2_hyperoptpars:[2][2][0.1, 0.3]1_model_hyperopt_package_2020-04-22_23:09:03.model'

with open(model_package_name, 'rb') as file_pi:
    model_package = pickle.load(file_pi)

model = model_package['model']

yhat = model.predict(X)

scaler = model_package['scaler']

print(yhat)

inv_yhat = scaler.inverse_transform(yhat)
print(inv_yhat)



# -

'''
# por que reshapeamos aca?
test_X_2d = test_X.reshape((test_X.shape[0], n_past_steps*n_features))
train_X_2d = train_X.reshape((train_X.shape[0], n_past_steps*n_features))
# invert scaling for forecast
# test
# Por que lo que hace es armar una matrix n_samples,6 con una fila por cada datapoint con la primera columna con la prediccion y las otras las features?
inv_yhat = concatenate((yhat, test_X_2d[:, -1*(n_features-1):]), axis=1)
# afecta en algo aplicar invers_stransform sobre una sola columna que con tra la matriz?
inv_yhat = scaler.inverse_transform(inv_yhat)
inv_yhat = inv_yhat[:,0]
# train
tr_inv_yhat = concatenate((tr_yhat, train_X_2d[:, -1*(n_features-1):]), axis=1)
tr_inv_yhat = scaler.inverse_transform(tr_inv_yhat)
tr_inv_yhat = tr_inv_yhat[:,0]

# invert scaling for actual
#test
test_y_2d = test_y.reshape((len(test_y), 1))
inv_y = concatenate((test_y_2d, test_X_2d[:, -1*(n_features-1):]), axis=1)
inv_y = scaler.inverse_transform(inv_y)
inv_y = inv_y[:,0]
#train
train_y_2d = train_y.reshape((len(train_y), 1))
tr_inv_y = concatenate((train_y_2d, train_y_2d[:, -1*(n_features-1):]), axis=1)
tr_inv_y = scaler.inverse_transform(tr_inv_y)
tr_inv_y = tr_inv_y[:,0]
'''

aa = [1,2,3,4,5,6,7,8,9,10]
aa[-5:]

Prediction.objects.all()[:3]



StarcoderdataJupyterScriptsDedupFiltered