# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: nilmtk-env
#     language: python
#     name: nilmtk-env
# ---

# + [markdown] toc="true"
# # Table of Contents
#  <p><div class="lev1"><a href="#Prepare-the-data"><span class="toc-item-num">1 - </span>Prepare the data</a></div><div class="lev1"><a href="#Run-an-easy-ConvNet"><span class="toc-item-num">2 - </span>Run an easy ConvNet</a></div><div class="lev2"><a href="#Using-the-functional-API"><span class="toc-item-num">2.1 - </span>Using the functional API</a></div><div class="lev1"><a href="#Run-J.-Kelly-ConvNet"><span class="toc-item-num">3 - </span>Run J. Kelly ConvNet</a></div>
# -

# # Prepare the data

# +
from __future__ import print_function
from matplotlib.pylab import plt
from matplotlib import rcParams
rcParams['figure.figsize'] = (13, 6)
plt.style.use('ggplot')
import nilmtk
from nilmtk.utils import print_dict
from nilmtk import DataSet
from neuralnilm.data.loadactivations import load_nilmtk_activations
from neuralnilm.data.syntheticaggregatesource import SyntheticAggregateSource
from neuralnilm.data.realaggregatesource import RealAggregateSource
from neuralnilm.data.stridesource import StrideSource
from neuralnilm.data.datapipeline import DataPipeline
from neuralnilm.data.processing import DivideBy, IndependentlyCenter

from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution1D
from keras.optimizers import SGD
import time


# create dictionary with train, unseen_house, unseen_appliance
def select_windows(train_buildings, unseen_buildings):
    windows = {fold: {} for fold in DATA_FOLD_NAMES}

    def copy_window(fold, i):
        windows[fold][i] = WINDOWS[fold][i]

    for i in train_buildings:
        copy_window('train', i)
        copy_window('unseen_activations_of_seen_appliances', i)
    for i in unseen_buildings:
        copy_window('unseen_appliances', i)
    return windows


def filter_activations(windows, activations):
    new_activations = {
        fold: {appliance: {} for appliance in APPLIANCES}
        for fold in DATA_FOLD_NAMES}
    for fold, appliances in activations.iteritems():
        for appliance, buildings in appliances.iteritems():
            required_building_ids = windows[fold].keys()
            required_building_names = [
                'UK-DALE_building_{}'.format(i) for i in required_building_ids]
            for building_name in required_building_names:
                try:
                    new_activations[fold][appliance][building_name] = (
                        activations[fold][appliance][building_name])
                except KeyError:
                    pass
    return activations    



NILMTK_FILENAME = './redd_data/redd.h5'
SAMPLE_PERIOD = 6
STRIDE = None
APPLIANCES = ['fridge']
WINDOWS = {
    'train': {
        1: ("2011-04-19", "2011-05-21"),
        2: ("2011-04-19", "2013-05-01"),
        3: ("2011-04-19", "2013-05-26"),
        6: ("2011-05-22", "2011-06-14"),
    },
    'unseen_activations_of_seen_appliances': {
        1: ("2011-04-19", None),
        2: ("2011-04-19", None),
        3: ("2011-04-19", None),
        6: ("2011-05-22", None),
    },
    'unseen_appliances': {
        5: ("2011-04-19", None)
    }
}

# get the dictionary of activations for each appliance
activations = load_nilmtk_activations(
    appliances=APPLIANCES,
    filename=NILMTK_FILENAME,
    sample_period=SAMPLE_PERIOD,
    windows=WINDOWS
)

# get pipeline for the fridge example
num_seq_per_batch = 64
target_appliance = 'fridge'
seq_length = 512
train_buildings = [1, 2, 3, 6]
unseen_buildings = [5]
DATA_FOLD_NAMES = (
    'train', 'unseen_appliances', 'unseen_activations_of_seen_appliances')

filtered_windows = select_windows(train_buildings, unseen_buildings)
filtered_activations = filter_activations(filtered_windows, activations)

synthetic_agg_source = SyntheticAggregateSource(
    activations=filtered_activations,
    target_appliance=target_appliance,
    seq_length=seq_length,
    sample_period=SAMPLE_PERIOD
)

real_agg_source = RealAggregateSource(
    activations=filtered_activations,
    target_appliance=target_appliance,
    seq_length=seq_length,
    filename=NILMTK_FILENAME,
    windows=filtered_windows,
    sample_period=SAMPLE_PERIOD
)


# ------------
# needed to rescale the input aggregated data
# rescaling is done using the a first batch of num_seq_per_batch sequences
sample = real_agg_source.get_batch(num_seq_per_batch=1024).next()
sample = sample.before_processing
input_std = sample.input.flatten().std()
target_std = sample.target.flatten().std()
# ------------



pipeline = DataPipeline(
    [synthetic_agg_source, real_agg_source],
    num_seq_per_batch=num_seq_per_batch,
    input_processing=[DivideBy(input_std), IndependentlyCenter()],
    target_processing=[DivideBy(target_std)]
)
# -

# # Run an easy ConvNet

# +
# get the shape of X_train
(X_train, Y_train) = pipeline.train_generator().next()


starting_time = time.time()
nb_epoch = 1
# define the network architecture = Conv Net
model = Sequential()
model.add(Convolution1D(nb_filter = 16, filter_length = 3, border_mode='same',
                        input_shape = (1, X_train.shape[2]), 
                        init = 'normal', activation =  'relu'))
model.add(Flatten())
model.add(Dense(4080))
model.add(Activation('relu'))
model.add(Dense(3))
# compile the model
# sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error',
              optimizer='rmsprop')
compiling_time = time.time() - starting_time
print('compiling time = ', compiling_time)
history = model.fit_generator(pipeline.train_generator(fold = 'train'), \
                    samples_per_epoch = 64064, \
                    nb_epoch = 2, verbose = 2)
print('run time = ', time.time() - starting_time)
# -

# (x_test, y_test) = pipeline.train_generator(fold = 'unseen_appliances').next()
res = model.predict_generator(pipeline.train_generator(fold = 'unseen_appliances', source_id = 0),
                              val_samples = 64*10)

from neuralnilm.metrics import Metrics
score = Metrics(state_boundaries = [2.5])
results = score.compute_metrics(y_pred, y_test)
results

# ## Using the functional API

# get the shape of X_train
(X_train, Y_train) = pipeline.train_generator().next()
print(X_train.shape)

# +
from keras.layers import Input, Dense, Flatten
from keras.models import Model

starting_time = time.time()

# define the network architecture = Conv Net
input_seq = Input(shape = (1, X_train.shape[2]))
conv1_layer =  Convolution1D(nb_filter = 16, filter_length = 3, border_mode='same',
                      init = 'normal', activation =  'relu')
conv1 = conv1_layer(input_seq)
flat = Flatten()(conv1)
dense1 = Dense(4080, activation = 'relu')(flat)
predictions = Dense(3, activation = 'relu')(dense1)
# create the model
model = Model(input=input_seq, output=predictions)
# compile the model
# sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error',
              optimizer='rmsprop')
compiling_time = time.time() - starting_time
print('compiling time = ', compiling_time)
history = model.fit_generator(pipeline.train_generator(fold = ' train'), \
                    samples_per_epoch = 64064, \
                    nb_epoch = 2, verbose = 2)
print('run time = ', time.time() - starting_time)
# -

model.evaluate_generator(pipeline.train_generator(fold = 'unseen_appliances'), val_samples = 64)

# # Run <NAME> ConvNet

# +
# get the shape of X_train
(X_train, Y_train) = pipeline.train_generator().next()


starting_time = time.time()
# define the network architecture = Conv Net
model = Sequential()
model.add(Convolution1D(nb_filter = 16, filter_length = 3, subsample_length = 1,
                        border_mode='same',
                        input_shape = (1, X_train.shape[2]), 
                        init = 'uniform', activation =  'linear'))
model.add(Convolution1D(nb_filter = 16, filter_length = 4, subsample_length = 1,
                        border_mode='same', 
                        init = 'uniform', activation =  'linear'))
model.add(Flatten())
model.add(Dense(4096, activation='relu'))
model.add(Dense(3072, activation='relu'))
model.add(Dense(2048, activation='relu'))
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dense(3))
model.add(Activation('linear'))
# compile the model
# sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error',
              optimizer='rmsprop')
compiling_time = time.time() - starting_time
print('compiling time = ', compiling_time)
history = model.fit_generator(pipeline.train_generator(fold = ' train', source_id = 0), \
                    samples_per_epoch = 64064, \
                    nb_epoch = 2, max_q_size = 50, verbose = 2)
print('run time = ', time.time() - starting_time)
# -

model.get_output_shape_at[0]

model.evaluate_generator(pipeline.train_generator(fold = ' train', source_id = 1), val_samples = 500, max_q_size = 5)

y_pred =  model.predict_on_batch(x_test)

# %matplotlib inline
plt.plot(y_test[:,0], y_pred[:,0], 'o');



StarcoderdataJupyterScriptsDedupFiltered