# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + colab={} colab_type="code" executionInfo={"elapsed": 911, "status": "ok", "timestamp": 1595151496938, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgTlAXb28EMHx7oSDz2IlYWnQTswhUuy-nr1hxDuA=s64", "userId": "16918877143175015576"}, "user_tz": -120} id="Km6Lind_Tajp"
from __future__ import print_function, division

import numpy as np
from tensorflow.keras.datasets import mnist, fashion_mnist, cifar10
from tensorflow.keras.models import Model, load_model
from tensorflow.keras import initializers, regularizers, constraints
from tensorflow.keras.utils import plot_model, to_categorical
from tensorflow.keras.layers import Input, Reshape, BatchNormalization, Concatenate, multiply
from tensorflow.keras.layers import Dense, Flatten, LeakyReLU, Dropout, Embedding, Activation, concatenate
from tensorflow.keras.layers import Conv2D, Conv2DTranspose
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.inception_v3 import preprocess_input
from tensorflow.keras.optimizers import Adam
from tensorflow.train import Checkpoint, CheckpointManager

import json
import matplotlib.pyplot as plt
from numpy.random import randn, randint
from skimage.transform import resize
import time
import random
import os
import tensorflow as tf
from sklearn.model_selection import KFold
from tensorboard.plugins.hparams import api as hp
from scipy.linalg import sqrtm
# -

from tensorflow.compat.v1 import ConfigProto, InteractiveSession
config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config)


# + colab={} colab_type="code" executionInfo={"elapsed": 1201, "status": "ok", "timestamp": 1595151497899, "user": {"displayName": "<NAME>", "photoUrl": "<KEY>", "userId": "16918877143175015576"}, "user_tz": -120} id="fAAomh8YyVuW"
def random_list(num_samples):
    randomlist = []

    for i in range(num_samples):
        n = random.randint(0,9)
        randomlist.append(n)
    return randomlist


# scale an array of images to a new size
def scale_images(images, new_shape):
    images_list = list()
    for image in images:
        # resize with nearest neighbor interpolation
        new_image = resize(image, new_shape, 0)
        # store
        images_list.append(new_image)
    return np.asarray(images_list)


# -

def set_checkpoint(generator,discriminator, cGan,max_to_keep=10):
    
    checkpoint_path = 'C:/Users/simoc/Documents/checkpoint_aml'
    ckpt = Checkpoint( generator= generator, discriminator = discriminator, cGan= cGan)
    ckpt_manager = CheckpointManager(ckpt, checkpoint_path, max_to_keep=max_to_keep)
    
    # if a checkpoint exists, restore the latest checkpoint.
    if ckpt_manager.latest_checkpoint:
        ckpt.restore(ckpt_manager.latest_checkpoint)
        print('Latest checkpoint restored!!')

    return ckpt, ckpt_manager


# ## MODEL EVALUATION- FRECHET JOINT DISTANCE

# +
def get_alpha(act, one_hot_cond):
    
    act_norm = np.mean(np.linalg.norm(act, axis=1))
    cond_norm = np.mean(np.linalg.norm(one_hot_cond, axis=1))
    
    return act_norm / cond_norm



def FJD(inception, images1, images2, class_labels, alpha=None):
    
    # calculate activations
    act1 = inception.predict(images1)
    act2 = inception.predict(images2)
    
    sampled_labels = np.array(class_labels).reshape(-1, 1)
    sampled_labels_categorical = to_categorical(sampled_labels)

    if alpha == None:
        alpha = get_alpha(act2, sampled_labels_categorical)
    
    joint_embed_1 = np.concatenate([act1, alpha*sampled_labels_categorical], axis=1)
    joint_embed_2 = np.concatenate( [act2, alpha*sampled_labels_categorical], axis=1)
    

    mu1, sigma1 = np.mean(joint_embed_1, axis=0), np.cov(joint_embed_1, rowvar=False)
    mu2, sigma2 = np.mean(joint_embed_2, axis=0), np.cov(joint_embed_2, rowvar=False)

    # calculate sum squared difference between means
    ssdiff = np.sum((mu1 - mu2)**2.0)
    # calculate sqrt of product between cov
    covmean = sqrtm(sigma1.dot(sigma2))
    # check and correct imaginary numbers from sqrt
    if np.iscomplexobj(covmean):
        covmean = covmean.real
    # calculate score
    fid = ssdiff + np.trace(sigma1 + sigma2 - 2.0 * covmean)

    return fid



def evaluate_FJD(generator, X_test, inception, class_labels, alpha = None ):
    
    X_test = X_test[:50]
    
    noise = generate_latent_points(50)
    
    sampled_labels = np.array(class_labels).reshape(-1, 1)
    sampled_labels_categorical = to_categorical(sampled_labels)
    X_fake = generator.predict([noise, sampled_labels_categorical])
    
   # X_fake = X_train.astype('float32')
    #X_test = X_test.astype('float32')
    
    X_fake = scale_images(X_fake,(299,299,3))
    X_test = scale_images(X_test,(299,299,3))
    
    X_fake = preprocess_input(X_fake)
    X_test = preprocess_input(X_test)
    
    fjd = FJD(inception, X_fake, X_test, class_labels, alpha)
    
    return fjd


# -

def FID(model, images1, images2):
    
    # calculate activations
    act1 = model.predict(images1)
    act2 = model.predict(images2) 

    #calculate mean and covariance statistics
    mu1, sigma1 = act1.mean(axis=0), np.cov(act1, rowvar=False)
    mu2, sigma2 = act2.mean(axis=0), np.cov(act2, rowvar=False)
    
    # calculate sum squared difference between means
    ssdiff = np.sum((mu1 - mu2)**2.0)
    # calculate sqrt of product between cov
    covmean = sqrtm(sigma1.dot(sigma2))
    # check and correct imaginary numbers from sqrt
    if np.iscomplexobj(covmean):
        covmean = covmean.real
    # calculate score
    fid = ssdiff + np.trace(sigma1 + sigma2 - 2.0 * covmean)
    return fid



# + colab={} colab_type="code" executionInfo={"elapsed": 997, "status": "ok", "timestamp": 1595151497901, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgTlAXb28EMHx7oSDz2IlYWnQTswhUuy-nr1hxDuA=s64", "userId": "16918877143175015576"}, "user_tz": -120} id="QX2vvt0eTajs"
def load_dataset(num_classes=10, dataset='mnist'):
    
    if dataset == 'mnist':
        (X_train, y_train), (X_test, y_test) = mnist.load_data()
    elif dataset == 'fashion_mnist':
        (X_train, y_train), (X_test, y_test) = fashion_mnist.load_data()
    elif dataset == 'cifar10':
        (X_train, y_train), (X_test, y_test) = cifar10.load_data()
    else:
        raise ValueError('Dataset Not Found!')

    
    X_train = np.expand_dims(X_train, axis = 3)
    X_test = np.expand_dims(X_test, axis = 3)
    
    print('X_train, y_train shape:', X_train.shape, y_train.shape, 
          '\nX_test shape, y_test shape:', X_test.shape, y_test.shape)
    
    # convert class vectors to binary class matrices 
    y_train = to_categorical(y_train, num_classes=num_classes)
    y_test = to_categorical(y_test, num_classes=num_classes)
    
    # the generator is using tanh activation in the last layer, for which we need to preprocess 
    # the image data into the range between -1 and 1.

    X_train = np.float32(X_train)

    # clip in range [-1, 1] 
    X_train = (X_train / 127.5)-1


    X_test = np.float32(X_test)
    # clip in range [-1, 1] 
    X_test = (X_test/ 127.5) -1


    print('y_train reshape:', y_train.shape)
    print('y_test reshape:', y_test.shape)
    return X_train, y_train,  X_test, y_test



def plot_loss(d_loss,g_loss, dim):
    plt.close()
    plt.figure(figsize=dim)
    plt.plot(d_loss)
    plt.plot(g_loss)
    plt.title('Model loss')
    plt.ylabel('Loss')
    plt.xlabel('Iteration')
    plt.legend(['Discriminator', 'Adversarial'], loc='upper right')
    plt.show()

def evaluate(generator, X_test, inception, it ):
    
    np.random.shuffle(X_test)
    X_test = X_test[:50]
    
    noise = generate_latent_points(50)
    sampled_labels = np.array(class_labels).reshape(-1, 1)
    sampled_labels_categorical = to_categorical(sampled_labels)
    X_fake = generator.predict([noise, sampled_labels_categorical])

    X_fake = scale_images(X_fake,(299,299,3))
    X_test = scale_images(X_test,(299,299,3))
    
    X_fake = preprocess_input(X_fake)
    X_test = preprocess_input(X_test)
    
    fid = FID(inception, X_fake, X_test)
    return fid

def plot_fjd(fjd,it):
    
    plt.plot(it,fjd)
    plt.title('Frechet Joint Distance')
    plt.ylabel('FJD')
    plt.xlabel('Iteration')
    plt.grid()
    plt.show()

def generate_latent_points(n_samples, latent_dim=100):
    x_input = randn(latent_dim * n_samples)
    # reshape into a batch of inputs for the network
    x_input = x_input.reshape((n_samples, latent_dim)) 	
    return x_input


#plot result

def plot_real_samples(X_train,num_samples, class_labels, title="", r=5, c=10):
    
    fig=plt.figure(figsize=(9,7), tight_layout=True)
    
    sampled_labels = np.array(class_labels).reshape(-1, 1)
    sampled_labels_categorical = to_categorical(sampled_labels)
    n=1
    for k in range(num_samples):
        plt.subplot(r, c, n, xticks=[], yticks=[])
        aux = np.squeeze(X_train[k])
        plt.imshow(((aux + 1)* 127).astype(np.uint8),cmap='gray')
        n+=1

    fig.suptitle(title,y=1.01)
    plt.show()

def plot_result(generator, class_labels, title="", num_samples=25, r=5, c=10, fashion_mnist=False):
        
        noise = generate_latent_points(num_samples)
        
        sampled_labels = np.array(class_labels).reshape(-1, 1)
        sampled_labels_categorical = to_categorical(sampled_labels)
        x_fake = generator.predict([noise, sampled_labels_categorical])

        plt.figure(figsize=(10,7), tight_layout=True)
        n=1
        for i in range(num_samples):
            plt.subplot(r, c, n, xticks=[], yticks=[])
            plt.imshow(x_fake[i, :, :, 0], cmap='gray')
            if fashion_mnist == True:
                class_labels_fashion = zip( class_labels, ['T-shirt/top','Trouser','Pullover','Dress', 'Coat','Sandal','Shirt','Sneaker','Bag','Ankle boot']*5)
                class_labels_fashion = list(class_labels_fashion)
                plt.title('{}'.format(class_labels_fashion[i][1] ))
            else:
                plt.title("Digit: %d" % sampled_labels[i])
            n+=1      
        plt.suptitle(title, y=1.01)


# -

# ## Conditional GANs model

# + colab={} colab_type="code" executionInfo={"elapsed": 1642, "status": "ok", "timestamp": 1595151500263, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgTlAXb28EMHx7oSDz2IlYWnQTswhUuy-nr1hxDuA=s64", "userId": "16918877143175015576"}, "user_tz": -120} id="kOioJxymXdQj"
class CGAN:
    
    def __init__(self, img_width, img_height, n_channels, n_classes):
        
        self.img_width = img_width
        self.img_height = img_height
        self.n_channels = n_channels
        self.img_shape = (self.img_width, self.img_height, self.n_channels)
        self.n_classes = n_classes
        self.latent_dim = 100

        self.optimizer = Adam(lr=0.001)

    
    def build_discriminator_model(self, input_shape=(28,28,1)):

        rn = initializers.RandomNormal(stddev=0.02)
        
        model_input = Input(shape= input_shape, name='discriminator_input')

        x = model_input

        labels = Input(shape=(self.n_classes,))
        labels_embedded = Flatten()(Embedding(self.n_classes, self.latent_dim)(labels))
        labels_embedded = Dense(self.img_width * self.img_width)(labels)
        labels_embedded = Reshape((self.img_width, self.img_height, self.n_channels))(labels_embedded)

        x = concatenate([x, labels_embedded])


        x = Conv2D(filters=56, kernel_size=(5,5), strides=2, padding='same',kernel_initializer=rn)(x)
        x = LeakyReLU(alpha=0.2)(x)

        x = Conv2D(filters=112, kernel_size=(5,5), strides=2, padding='same',kernel_initializer=rn)(x)
        x = LeakyReLU(alpha=0.2)(x)

        x = Conv2D(filters=224, kernel_size=5, strides=2, padding='same', kernel_initializer=rn)(x)
        x = LeakyReLU(alpha=0.2)(x)


        x = Flatten()(x)
        x = Dense(1)(x)
        x = Activation('sigmoid')(x)
        # model_input is conditioned by labels
        discriminator = Model(inputs=[model_input, labels], outputs=x, name='discriminator')

        return discriminator

    def build_generator(self):
        
        rn = initializers.RandomNormal(stddev=0.02)

        inputs = Input(shape=(self.latent_dim,), name='z_input')
        labels = Input(shape=(self.n_classes,), name='class_labels')
        

        x = concatenate([inputs, labels], axis=1)

        x = Dense(7*7*112)(x)
        x = Reshape((7, 7, 112))(x)
        x = BatchNormalization()(x)
        x = Activation(LeakyReLU(0.2))(x)
        
        x = Conv2DTranspose(filters=56, kernel_size=(5,5), strides=(2,2), padding='same',  kernel_initializer=rn)(x)
        x = BatchNormalization()(x)
        x = Activation(LeakyReLU(0.2))(x)
        
        x = Conv2DTranspose(filters=64, kernel_size=(5,5), strides=(2,2), padding='same', kernel_initializer=rn)(x)
        x = BatchNormalization()(x)
        x = Activation(LeakyReLU(0.2))(x)
        x = Conv2DTranspose(filters=1, kernel_size=(5,5), strides=(1,1), padding='same',kernel_initializer=rn)(x)

        x = Activation('tanh')(x)
        # input is conditioned by labels
        generator = Model(inputs=[inputs, labels], outputs=x, name='generator')
        return generator

    def cgan(self, generator, discriminator):
         # Build and compile the discriminator
        discriminator.compile(loss=['binary_crossentropy'],
                                   optimizer=Adam(lr=0.0002, beta_1=0.5),
                                   metrics=['binary_accuracy'])
        discriminator.trainable = False
        #discriminator.summary()

        #generator.summary()

        noise = Input(shape=(self.latent_dim, ))
        label = Input(shape=(self.n_classes,))
        img = generator([noise, label])

        # during generator updating,  the discriminator is fixed (will not be updated).
       

        # The discriminator takes generated image and label as input and determines its validity
        validity = discriminator([img, label])

        cgan_model = Model(inputs=[noise, label], outputs=validity)
        cgan_model.compile(loss=['binary_crossentropy'],
                                optimizer=Adam(lr=0.0002, beta_1=0.5),
                                metrics=['binary_accuracy'])
        
        plot_model(cgan_model, show_shapes=True, to_file='cgan-adversarial_model.png')
        plot_model(generator, show_shapes=True, to_file='cgan-generator_model.png')
        plot_model(discriminator, show_shapes=True, to_file='cgan-discriminator.png')

        return cgan_model



    def train(self, x_train, x_test, y_train, generator, discriminator, cGAN, 
              class_labels, iterations=100000, check_point_interval=25000, batch_size=32, 
              sample_interval=300, k=1, fashion_mnist=False, save_model=False, verbose=True):
        
        # Adversarial ground truths
        real = np.ones(shape=(batch_size, 1))
        fake = np.zeros(shape=(batch_size, 1))
        
        inception = InceptionV3(include_top=False, pooling='avg', input_shape=(299,299,3))

        # Label smoothing (label from 1 to 0.9: it helps during training)
        smooth=0.1

        init_time = time.time()

        d_loss = []
        g_loss = []

        tot_time = 0
        time_tmp = 0
        x_fjd = []
        fjd_list = []
        ckpt, ckpt_manager = set_checkpoint(generator=generator,discriminator=discriminator, cGan=cGAN)
        
        for it in range(iterations):
            start = time.time()
            
            for _ in range(k):
                
                discriminator.trainable = True
                # Select a random half batch of images
                idx = np.random.randint(0, x_train.shape[0], size=batch_size)
                imgs, labels = x_train[idx], y_train[idx]


                # Condition on labels (random one-hot labels)
                fake_labels = np.eye(self.n_classes)[np.random.choice(self.n_classes, batch_size)]
                
                # Generate a half batch of new images
                # Generate sample noise for generator input
                z = generate_latent_points(n_samples= batch_size)
            
                # we can use labels instead of fake_labels; because it is fake for noise
                x_fake = generator.predict([z, labels])

                # --------------------- Train the Discriminator ---------------------
                # Update the Discriminator by ascending its stochastic gradient
                d_loss_real = discriminator.train_on_batch([imgs, labels], real*(1-smooth))
                d_loss_fake = discriminator.train_on_batch([x_fake, fake_labels], fake)
                
                # Discriminator loss
                d_loss_batch = 0.5*np.add(d_loss_fake, d_loss_real)
                discriminator.trainable = False

            #  --------------------- Train the Generator ---------------------

            z = generate_latent_points(n_samples= batch_size)
            fake_labels = np.eye(self.n_classes)[np.random.choice(self.n_classes, batch_size)]
            # Train the generator
            g_loss_batch= cGAN.train_on_batch([z, fake_labels], real)

            d_loss.append(d_loss_batch[0])
            g_loss.append(g_loss_batch[0])

            end= time.time()
            time_tmp += ( end-start )
            # If at save interval => save generated image samples
            if it % sample_interval == 0:
                
                fjd= evaluate_FJD(generator, x_test, inception, class_labels)
                fjd_list.append(fjd)
                print('\n------------------------------------------------------------- \n')
                print('\nIteration {0}/{1} :  d_loss={2}, g_loss={3}, FJD={4}\n'.format(it , iterations, d_loss[-1], g_loss[-1], fjd))
               
                
                hours, rem = divmod(time_tmp, 3600)
                minutes, seconds = divmod(rem, 60)
                print("\nExecution Time for {} terations: {:0>2}:{:0>2}:{:05.2f}\n".format(sample_interval, int(hours),int(minutes),seconds),end="\n\n")
                print('\n-------------------------------------------------------------\n')                                                                         
                
                tot_time+= (time_tmp)
                
                hours_, rem_ = divmod(tot_time, 3600)
                minutes_, seconds_ = divmod(rem_, 60)
                print("\n Total time  for {} terations: {:0>2}:{:0>2}:{:05.2f}\n".format(it, int(hours_),int(minutes_),seconds_),end="\n\n")
                print('\n-------------------------------------------------------------\n')                                                                         
                
                
                if verbose == True:
                    self.plot_generated_images(generator=generator, class_labels=class_labels, tot_time=tot_time, 
                                           it=it, fashion_mnist=fashion_mnist)
                    plot_loss(d_loss, g_loss, (6,4)) 
                    
                    x_fjd.append(it)
                    
                    plot_fjd(fjd_list,x_fjd)
                time_tmp = 0
            
            if save_model == True:
                if it % check_point_interval == 0:
                    ckpt_save_path = ckpt_manager.save()
                    print('Saving checkpoint for iteration {} at {}'.format(it,ckpt_save_path))
                                                                                
                
        if save_model == True:
            print('saving model ...')
            generator.save('model/generator')
            disctiminator.save('model/discriminator')
            cGAN.save('model/cGAN')
        
        return d_loss, g_loss, fjd_list, tot_time         

    def plot_generated_images(self, generator, class_labels, tot_time, it, fashion_mnist, r=5, c=10):
        
        num_samples = len(class_labels)
        noise = generate_latent_points(num_samples)
        
        sampled_labels = np.array(class_labels).reshape(-1, 1)
        sampled_labels_categorical = to_categorical(sampled_labels)
        x_fake = generator.predict([noise, sampled_labels_categorical])

        plt.figure(figsize=(13,9))

        n=1
        for i in range(num_samples):
            plt.subplot(r, c, n, xticks=[], yticks=[])
            plt.imshow(x_fake[i, :, :, 0], cmap='gray')
            if fashion_mnist == True:
                class_labels_fashion = zip( class_labels, ['T-shirt/top','Trouser','Pullover','Dress', 'Coat','Sandal','Shirt','Sneaker','Bag','Ankle boot']*5)
                class_labels_fashion = list(class_labels_fashion)
                plt.title('{}'.format(class_labels_fashion[i][1] ))
            else:
                plt.title("Digit: %d" % sampled_labels[i])
            n+=1      
        
        hours, rem = divmod(tot_time, 3600)
        minutes, seconds = divmod(rem, 60)
                
        plt.suptitle("Iteration {}: \n Execution Time: {:0>2}:{:0>2}:{:05.2f}".format(it, int(hours),
                                                                                      int(minutes),seconds), y=0.97)

        plt.show()
    
    
    


# +
def crossvalidation(X_train, y_train, batch_size_list,  class_labels, save_name='crossvalidation_result.txt', iterations=100000, fashion_mnist=False):
        
    kfold = KFold(n_splits = 2)
        
    fid_score= []
    
    c = True
    for batch in batch_size_list:
            
        fid_tmp = []    
        fold=0
        
        for train_index, test_index in kfold.split(X_train):
                
            print('\n------------------Batch_size {0}-Fold {1}----------------\n'.format(batch,fold))
            fold+=1
            if c ==True:
                c=False
                continue
                
            Xtr=X_train[train_index]
            Xte=X_train[test_index]
            Xte = np.repeat(Xte, 3, 3)
            

            Yte=y_train[test_index]
            Ytr=y_train[train_index]
                
            cgan = CGAN(img_w, img_h, num_channels, num_classes)
            discriminator = cgan.build_discriminator_model()
            generator = cgan.build_generator()
            cGAN= cgan.cgan(generator, discriminator)
                
            d_loss, g_loss, fjd_list, tot_time = cgan.train(x_train=Xtr, x_test=Xte, y_train=Ytr, generator=generator, 
                                                            discriminator=discriminator, cGAN=cGAN, 
                                                            class_labels=class_labels,iterations=iterations, 
                                                            check_point_interval=25000, 
                                                            batch_size=batch, sample_interval=500, 
                                                            k=1, fashion_mnist=fashion_mnist, verbose=True)
                
                
            cgan.plot_generated_images(generator=generator, class_labels=class_labels, tot_time=tot_time, 
                                        it=iterations, fashion_mnist=fashion_mnist)
                
            plot_loss(d_loss, g_loss, (6,4)) 
            time_tmp = 0
                
                
            fid_tmp.append( fjd_list[-1])
                
            

        fid_score.append(np.mean(fid_tmp))
            
        with open(save_name, 'w') as f:
            f.write(json.dumps(list( zip(batch_size_list, fid_score))))
                
    return fid_score
            
            
        
# -

# ## Crossvalidation FASHION MNIST

#
# ### Class Labels Fashion Mnist:
#
# ![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAANgAAAG0CAYAAABQYel4AAAgAElEQVR4Ae2dz4scR5bH94+pSyOog4UPY1/MMGAGg7APEoZFsGIKgdQMjGBgPDeJXtRC3fhgY8MaBmawwEiChhVjGLPDDmuw2gapaTBGYGwWyVokJFiwYEUhBG+JX5kRkZFVlS9fRmVmfwVNVWVFvsj8ft8nXkRUq+ufCP+gABToTIF/6iwyAkMBKEArA/bfD/6H8AMNjnoONB0zGgHWNPhRaq8SD//GrQDHYwAmlBMc8YW6RphMCnA8BmBC5nDEF+oaYTIpwPEYgAmZwxFfqGuEyaQAx2MA<KEY>//A/GllQwSbv/oV+8K7pp71NA/HZvWLHsYAhWoO54xpiIpp/ddmc+9YuHfyfdxMP92imZzoX6Nb/muPu3EATNcgonTZ+bQYdL0TTpxyPewQYET37kW7v/YXev3SBTpwspyxKMLMV7AxPiOWmULbtwQd25PKnId7zmTbWxRsAYPMv6aK+/vfoC1XQDz+0lcdUgmJr392j0mH+DV3USfg6nd7Zo4OH4RT6u38zg835v7oZQpxyTp8pVdosAKyccdh4xbXv0oE95GAwPpT9uuMOMOdjJSYR3b5i7t295841g2cZ07VzMct3mj0bNmBuRDr+SzNV+exzuv3nC2aeXwEsAYQbrR1gdpfq/J+/odtfVX9MsrkESsRrpr3+i1sNTwmaL61gd3cNUCc/ox/UmS7Bf/eX5P3dPnyo167z7/fooptCb0zplXcv0637pmuXkPWJ5/SZVXcvXf/FdJOoPp6rvgzArI+pa3SaOUjr+q87HhiwwosBAzanf1wyo9Ef/9MbZZ2JFcBO0ac/Roq4BLQffDpRKyNvcJpLoJ4D9vKh/Q2WKblkKgH73E6fgxurvJg/vEe33Hb/r3bpYF4C4SpA5aRFU2jnTQqwr6NIbvA7/mHzCmYBS12jq0yuYjnPYxjrjkdXufTlgAFziT6l7cKccl1VnSJO6c0d7wNOlYDnzZTw5GdmMVH8pkE8d3/5jA7+9k005ewxYP97jz79ndnEOXbeW289+ztd0NNBu3ngpcfPh3+n22qD7fHDcH1GD+nTf1YDmblff31zOzlLdL4k9FkAWHCdVProf8Tgkv50tEZ2xx0k/jUm12DH36MvojWYO9dJEsd0x5s+DgMw/zcV7G8s/PFvT+mHP6kdP7VTNKOLH3xI278/RSfesscSFezku6/SK++q3bPLxXbw5K0P9cishfOgK3afXNtKvEQCNVSfI77fhZvuqG304jc53McLamr3u8/oB3+RT0TF5sGG2wF0u3b2fhQEbufNaqp/M+Tdz8wmxMuHdMvCO3FT85pdxMrvfi4A7OS7p6037nqmNDm+Sbe8TZLifo+fogtXdunCn82v0FVheFpeo7uXYhfxdTr/7+XuTPVco3DdcV//VZ5zPM6/yeEW4d6jnva8fEhfXDpttl+P/5JmH39DT9wuYgqI73+kG5fM5zI6OYLPt6xcL5/S7Y8v0An7eYxqd+K3u3Sr2K1cMEKvorjXhiO+dzoVCefpcuyNU3T6Dx/SrW+T5UWf/uSr6udD23+9Z3ZYH39J28Hniqfo/M7nIagvn9F3e5fpdAGz+iztPbtOW6DPAsC2v35GB39yW+mv05u//YRue3DpC7d+F5/f/akOMPVxwzP6zv88c0PF9H00StaBVHfc13+V5xyPswG2yg0MuQ1H/CHfb+rapRI5FbsPxzgeAzAh5zjiC3XdmzAArGoFAKtqwjoCwMpdyXiTgSVoD0/ieAzAhIzkiC/UdW/CoIJVrQBgVU1YRwAYS7ZBncTxGIAJWcwRX6hrhMmkAMdjACZkDkd8oa4RJpMCHI8BmJA5HPGFukaYTApwPAZgQuZwxBfqGmEyKcDxGIAJmcMRX6hrhMmkAMdjACZkDkd8oa4RJpMCHI8BmJA5HPGFukaYTApwPG4EmOoAP9DgKOdAU5YbAdY0+FFqzxndjpI+Y7hXjscATMh5jvhCXSNMJgU4HgMwIXM44gt1jTCZFOB4DMCEzOGIL9Q1wmRSgOMxABMyhyO+UNcIk0kBjscATMgcjvhCXSNMJgU4HgMwIXM44gt1jTCZFOB4DMCEzOGIL9Q1wmRSgOMxABMyhyO+UNcIk0kBjscATMgcjvhCXSNMJgU4HgMwIXM44gt1jTCZFOB4DMCEzOGIL9Q1wmRSgOMxABMyhyO+UNcrhdF/8cn7ooaVTkKjQAGOx5kBc19jY7/TqviT2MF9DPIFR3x3o+7PnVW+46v4M9rl1/64c5o+rgqYbifoi/mT4O2vv+n9dtGe43FGwAxcxdfvuK/GETSzC1FXjckRPx071indqulRecDMdS7+I6Pmb9sXnuuvMWr/RRtN712qPcfjbIDpkSyGyX5v1GKTpOTpNg5H/PQVrRew9DWljq4AWAxU/DoVtsfHOB5nAiwayQoR7bd3jGBtwBG/kCF4Mh7AKlUTgAVOBy9aJdCCSiU95w8uOuOLVvoE18kHrPIVSN6MoUh264Vb7xXTN3sNsR/FzKM4b5N27beRuhj6sTJI+hXOfQ2SXXvbtWXYt2nvxwzfV18ouEcz++WB+jqLNeqUcsyCOB7nqWCeMEEuEZnvxfISIX5/KK854qfvjQdYdTPhKd3YcV8eX34xQ/ltoeX3PPvJmQZsk2YVj3yAqndSgOm/VZcH9rvGAqAc0D649vzZ2RAoN7D49+F3K/Wc43E/ANsY/i4TR/y08TzAigqVDmq/oDzWuTpFTwJmq0YYehFgNUuCJGA1bVVnGjxvU8RCF4BoLyq+7vBaZV5xPO4HYJXRUUaQnFE44qevrx4wN1L70yg3arv3Usmn+qkDME7M+LWJG4OpIi4ALAlSOMUr7r2urW4QabGgbf11Fj21fsLxOCNgYVl3dxsb6o4P7ZEjfvoeo6RKN0ofdV/rqtYm0aDVCrAolum8HrC6vvw1VHEDcZUq3lBPIi0AWKCO9yISqninOkUp3hrYk14AVmhm9J54U++6pI8HuPi1rgyNAKsHLwnYAmgaA5a8zkKU1k84HuepYG6KEgugxU1XttZqZA7AET99iXWDUbp17dEocXMBVg9kzRQxrlL+DcXVzeZLdRqcZ6DmeJwNMDN6TakUJ48ovl9dPueIn74eHmAHO+E6KV6TyAOW8s8cKz2O77Dm3uzUNjgvBZM9Nok2XfS9RcfiniVeczzOB5i6w0Ig+3mIvwUrocAaY3DET19uTRKmGxdHTZL5nzOFwMkD5nb5PC+1v96uX3F15RMDvjknBVRqA6c4u4hvNCrbhvdatBd+wvE4L2DCN9yncBzx+3T9EtdSB7FEbB2jAEwsYqNAHI8BWCOJ6xtzxK+PNsB37OzEfWzQyR0AsE5kHUTQIw9YDpcAWA6V+9kHAMvgCwDLIHJPuwBgPTVG8LI4HmMNJmQAR3yhrhEmkwIcjwGYkDkc8YW6RphMCnA8BmBC5nDEF+oaYTIpwPEYgAmZwxFfqGuEyaQAx2MAJmQOR3yhrhEmkwIcjwGYkDkc8YW6RphMCnA8BmBC5nDEF+oaYTIpwPG4EWCqA/xAg6OcA01ZbgTY8/kLwk9aA5V00CatzVh0UR43/QfAhAYNADZuuNQgAcCEYOGMuAAMgKWqGyqYEJQADIABMCGYUhUOgAEwAAbAsBHTIgewBmshXqoqNTmGCoYKhgrWIYAADIABMACGKWKLHMAUsYV4TaaDqbaoYKhgqGAdAgjAABgAA2CYIrbIAUwRW4iXmvY1OYYKhgrWqwq2f2VKW/vjMaUNYEqL8s9Ap55fpf01Dh5NBpoxtx1IBbtDW/a7dQFYaoAx+py5/gjTuZ4NKj0H7BFdm4WjMwADYEOqeD0HzIzMBir/eSrJhneszRQxTDJUsFCP/uRCzwHzhQJg9UmUBkyv067coef7V+16zV+XmXP8dVw4xbSzB3V+NO3ScWc36UFxvDrTCGO9oAfXzwVrxvj9xdfq58GwngOwIknyG9d1BTMgnKMzMSQWuCDJ79+kM2qdW7RdFbBEu/s3actbD+rr2PDgtn35/dde6xr9iQcWzmsAtkYDswDmJ7a+VwOEn9xF4mjwztG1+2qwSYBjtTIwuAq2ZGZhYYrXzqaildBVIFyjL4UeAtcAwARE5BqSBbBgKveCnuuEdxDFVdufaq4KmG23kY6pQYqvQWkewPyCQmjj6xruawA2dsCKKZ9N0iixw8GBA5iJayqQ2fH1q6N/3F/vmecllLpdfK1r9CbUhQ84AFujiVkqWJy0K1cwW1Xi8+dLqo1d37m1XG0Fi3QHYOXvdKzpb3IsmetHhkmNQF3GWQtgc79KRSNzVN100lemd+b8SeV4GSuAKopZpycAA2CV7eq6ZFn1+HoAc+ufKflTObM2i47ZauS30yCo3cYCsDu0FVS5eO3m1mjlhobWJ7XTGMQpgV1Vzz62wxRxjZVvbYCpe7a7e/66KN7pUwlrdvvK36ZRbcLKZiua/VU2HS8BSgFm0S4EDhVs7RVsHCOaP8rKATY+bXydhvx8QBVsfEkEwMbnaTwYALBRTBHHn6hx4g7lNQADYOKbN0NJ/hzXCcAAGADrMAcAWIfiLhshlfjL2uD9YU9/ARgAA+Qd5gAA61DcZdUHFWzY1WmZv+p9AAbAUME6zAEA1qG4y0Y4VDBUsPL3N8pna/pl3/GZAcDG52k8qKKCoYJhithhDnQOmOoAP9DgKOdAOflb7VmjKeJqIY9mK87odjSVGu5dczwGYEJ+c8QX6hphMinA8RiACZnDEV+oa4TJpADHYwAmZA5HfKGuESaTAhyPAZiQORzxhbpGmEwKcDwGYELmcMQX6hphMinA8RiACZnDEV+oa4TJpADHYwAmZA5HfKGuESaTAhyPAZiQORzxhbpGmEwKcDwGYELmcMQX6hphMinA8RiACZnDEV+oa4TJpADHYwAmZA5HfKGuESaTAhyPAZiQORzxhbpOhjnYmdLk7B49se8+2dukycYuHSRb4+AqCnA8zgqYMbn8082TncNV7msQbTjiBzd2dzf4Wlb3Z7Bne0+DZqu+AGCrKrV6O47H2QDTcHkjKj3eo5n62+YjgYwjfmCtBmyTbjz2jrbQCIB5Ogo95XicD7C7h8V0xd3vmKYtHPGdDvoxBRgRcTUCYIG6Ii84HmcDLHmHNUmVbNvzgxzxg1uq00JXsaiyBSemXwCwtC5tjnI8BmBtFPfO5YjvnU60EmCHtL0xpeq67CndOBtOt1cFrLIu9jdC7BR1+25wpfpFHJ/IXJtbO1Y2VIqBwl5r8j6q/fTpCMfjtQJWNalPcja7Fo74QQ81gIVTREnAXKKHO4vakw1XMavgmmuOrsOC6INv4nixLWCzsy52cPeDeMHxeG2AmcQZrthxRnDED2KkANPHplRWkCixiwBVEOLBKwSVbMX0AKiJVTlPtdPXVZ4b92VCRdeagLDociBPOB6vATA3cvqJMxCFF1wmR/wgnIWpnGKpjzPiAShK2iJAc8A0FDU7uCFU1T7Dc8375SDgLiq6JgtYtZ1r3/9HjseZATNmVObn/dd26RVyxA+CpipY0EC9qCa7aRIlMxHFVSWEptre7ypsG8WKQbGvw4Eh8VmnbhcPGH6v/X/O8TgfYM4I/7Ow/mu68hVyxA+CrwSYAcNf65gYVWAWA2ahWamCuSmhgUPDF3hYV8GCuyPzuScAi1QpX7ZNoNjwMvI4nrXVp3YXMZCnCpJ+2w1eHjCx3nFVil+X3aT6cGAf6t3KEPBU+zJa8QwVrJAi+aRdAq04yiV7HsbBdvqEVWLRHRsw/PWrTfDot2KWAUbkzis3K1S/+jx/q95ejOs3Ob2368cQPBXLiw3AFtlKrK9uKSO6tZc3N1cJ4X68kbc8Z1jPcgGmVDEQOP3UtKtaRZYDZvQNY4W/IBw4kKiSyfedp/HnXAAskKvyonUCVSKO6wD0GZefqbvheJxvkyN1xSM6xhF/RLd/JG6F4zEAE0oNjvhCXSNMJgU4HgMwIXM44gt1jTCZFOB4DMCEzOGIL9Q1wmRSgOMxABMyhyO+UNcIk0kBjscATMgcjvhCXSNMJgU4HgMwIXM44gt1jTCZFOB4DMCEzOGIL9Q1wmRSgOMxABMyhyO+UNcIk0kBjscATMgcjvhCXSNMJgU4HgMwIXM44gt1jTCZFOB43Agw1QF+oMFRzoGmLDcC7Pn8BeEnrYFKOmiT1mYsuiiPm/4DYEKDBgAbN1xqkABgQrBwRlwABsBS1Q0VTAhKAAbAAJgQTKkKB8AAGAADYNiIaZEDWIO1EC9VlZocQwVDBUMF6xBAAAbAABgAwxSxRQ5githCvCbTwVRbVDBUMFSwDgEEYAAMgAEwTBFb5ACmiC3ES037mhxDBUMFW28Fu3+Tznh/Vln92ewz1x+NZkRtDdj+1fJPiUc6be2PP3mbDGbratvrCvbg+lW6dt9LFJtQY4FMBrBzoUaqIhcD01XaX2OFXldS96nfXgOWEmr/ypQms5v0YASJ0xlgWptHdG02Hq1SuTCEY4MD7MH1cwDMDS66oicq2Krvu3Z47GzZMTjAdAW7cqczQXKOit1WMDW1vkNb/rpVTx0VkLa6+e/NX5AevLy1XHUqbuIVXyG1MaVyrVfGdO9Xz/em+0cE6kEBZhJgPOuK7gGzSe8GJAvYmVm16umBa8PT1q7jSkgiWBUg+1ctYFE/6r37N2lrRBtS3IG394AZ4+0Xx7lEGcnotx7AEjuxFqayGplKEwxoRfVLVSEDX3w+NynHdF7vAQvEdtvSIwFtXYDFINSua4M1npseelWuGOjc9LBaGQP/ivYpSMd5bFiAKYMC04dtSveARZWlpgoFswRvDWbWUj40DiQzo4hB9eOUU8the9R2gBgeYDXTmbZCrOP8zgHTg5FXcWoAq61gCyqOmT4mppvFIDilyUhmGm1yY3iAoYKVO6iLtLADUVBJagDjzgp0xaqBiANtm0Tu67k9BuwObcUfKKeSZsEo21fR3XV1VcFqq0sdYHM39fOqndLV3wn0n2vN/V3FO7QVgJbYVRywT84vzmOPAXtB/pzefbYSz/s5N92Xc2QAszus/topHphcctcCZtZJVb094Ozg5nxQj2V1dBsg3rUEwB3ddVivAesLCF1dR2vAHDh4LKfNPdMCgK3REAA2/soGwABYb0f/rmYOOeMCMAAGwDrMAQDWobjLRkpMETFFXO//aF5j8i+DQ+J9AAbAAFiHkAMwAAbAABjWaC1yAGuwFuK1nSaigqGCoYJ1CCAAA2AADIBhitgiBzBFbCEepojjr0ASHqeq1KJjjb5CVhGMH2hwlHNgEUyp9xoBlgqAY0YBzvQB2g1LAY7HAEzIY474Ql0jTCYFOB4DMCFzOOILdY0wmRTgeAzAhMzhiC/UNcJkUoDjMQATMocjvlDXCJNJAY7HAEzIHI74Ql0jTCYFOB4DMCFzOOILdY0wmRTgeAzAhMzhiC/UNcJkUoDjMQATMocjvlDXCJNJAY7HAEzIHI74Ql0jTCYFOB4DMCFzOOILdY0wmRTgeAzAhMzhiC/UNcJkUoDjMQATMocjvlDXCJNJAY7H6wPs7i6pP9k823uaSZ5uu+GIX3tFVhv/T1ur59t3a8/AGxkU4Hi8JsCe0o2z5m+fAzA/Mw5pW/9d+l068A8T0ZO9zc4AO9iZ0mTnMOoRL2MFhgOYN0IDMGejHXTWkOgAzHmw+HEggJlE2r5rRmsAZkxVFWqyUa1ciy2XeReArabjIADTiXR2j54QACttNYNOs8HGTSfLrxlKnW/ALdtMNjbpxmPbszeTKNd764G81KK/z/oP2OM9mhUGA7AilbQuDTYxUhtENka4ljqkbT2YFT2RrlZRpUQFK/VZ9KzngMWjNAArzGwEWKxjEYVIg+dVKO+t4mkwyJmjAKxQZ+GTXgOmTQxGUwBWuNkEsAQgRZy6aXdiKuhv+QOwUsFFz/oLWHJkBWClmQ20SGrpIkVxLLjB5kkCZgDm9Fv82FPAzJSmXET7C273fMm0ZvF99+Jdjvj+hVcrvP+u93zlCmZ1j7f9AZgnZrOnHI/X9EGzurFotG12r71rzRE/uAmb+KmdwKDdIt2C6pYGzO0qYooYqrrKK47HAGwVZVdowxG/EtatleKqQ6R3/woobLsAxgSguir6O4a2TfxrV+v8DK6iQY8PcDwGYEKGcsRPd103pY4+n/JgcdPvAsAicBRLbTLZ88K2ZjZh4kT9FLHwhOPxGgEbl2Ec8celwPjvhuMxABPKC474Ql0jTCYFOB4DMCFzOOILdY0wmRTgeAzAhMzhiC/UNcJkUoDjMQATMocjvlDXCJNJAY7HAEzIHI74Ql0jTCYFOB4DMCFzOOILdY0wmRTgeAzAhMzhiC/UNcJkUoDjMQATMocjvlDXCJNJAY7HAEzIHI74Ql0jTCYFOB4DMCFzOOILdY0wmRTgeAzAhMzhiC/UNcJkUoDjcSPAVAf4gQZHOQeastwIsOfzF4SftAYq6aBNWpux6KI8bvoPgAkNGgBs3HCpQQKACcHCGXEBGABLVTdUMCEoARgAA2BCMKUqHAADYAAMgGEjpkUOYA3WQrxUVWpyDBUMFQwVrEMAARgAA2AADFPEFjmAKWIL8ZpMB1NtUcFQwVDBOgQQgAEwAAbAMEVskQOYIrYQLzXta3JsPBXsDm1tTOnM9UcNYOScM7yK13PAjAnuzzyXj1dpf41gNIFoUVsZwFIanaNr93MmIwcWzjk570mmr0EAtrUvc7OLkn0d77UGbP8qqUEn1mf/CgBbh5+pPgHYGithO8Ae0bXZlCZX7jSYlnU1UHGqEeecrq6/u7j9Buz+TTqzkXs07k7seIRrB1ifEpRzLZxz8nkTe8V9DcAGW8Fe0P4V9W2fy9ejD66f01PJcg0bDVreQGZium8RTccO26gpagIWO30t+4ynsolz1ugFF6Bl5/UbsIRJ/ZgSyYyk7SqYugaTpCqJ63fw7tDW7CY98JK3AqYGzEBVrufsFDQ4N3UsdQ2qXQingdwHG4ClPgNTx9b3/8FcIgSmyyT7spGoi/fbA2bu3a8o9aB5OnkVS9+X1bVyrh7gPCj06xCched7ULvBoAQYgPUPMGWYTYbSKC9xAkP7f1wKMAd/AVpq4yMxGyg0jIFzOkZa6/ip2LaSVgC15/vTxLINAOsnYHVmuqQY0KM0YBo0C1IIT7RWi8Axg5ZXqZyGQbtFu5YxLOb1JNigSrcpgev/gOgGsiaPyuOm/9Y3RdTGx0YN15hOAJv7IPjPPZ0CcNysYBlgdlNlhQqmK11lGh/7Fr/2rs8BPoLH4QEWrwsGbEKXgJnKkAbM7SqGVW45YOa8xBrMVk1XjZKARW3cmsyd06QqDKltrwF7cP1q+Cs/bk6fHEWHNwK2A0xVgGqym3VYeTx+7dawal3UFDAHxSSoTqYS+TuZdTuGfhsXC4BVJ5DZpohupPUXyUVSDLhyuRG4PWDu8yrvMUh+NejYKrZh26j3mVNEc90lUMYXBXN1umfAdteValM9x+kypsdeV7AxCZ26l3aADa9ipzQY+zEAtsZKCMDGP0gAMADWg18WHi9oAAyAAbAOcwCAdSjusvUFpojjrVzOewAGwFDBOp/xTq0AABBSSURBVMwBANahuG4Uq3tEBUMFq34Kts7fpl8jDHWQtDkOwAAYAOsQagAGwAAYAMMarUUOYA3WQrw200N1LioYKhgqWIcAAjAA1howlUT4gQZHOQdSEC061ui36RcFOurvqaTDv3ErwPEYgAnlBEd8oa4RJpMCHI8BmJA5HPGFukaYTApwPAZgQuZwxBfqGmEyKcDxGIAJmcMRX6hrhMmkAMdjACZkDkd8oa4RJpMCHI8BmJA5HPGFukaYTApwPAZgQuZwxBfqGmEyKcDxGIAJmcMRX6hrhMmkAMdjACZkDkd8oa4RJpMCHI8BmJA5HPGFukaYTApwPAZgQuZwxBfqGmEyKcDxGIAJmcMRv3HXj/dotrFJNx43PhMnCCjA8XgNgB3StvvTz/pxHAnDEb/qeayN+nPVnj6rAqbbTWn7brUHHOErwPE4L2DW+NneU/5d9vRMjvjBrdzd1d+9HENxsNMxYKtCG1zs0XzB8TgjYGZ0HiNcKt044pdp+pRunJ3SZOewPJR61gUMXcRMXfsIjnE8zgeYHqF36WAEQqdugSN+GWfFwacLGLqIWd7YqJ5xPM4G2MHOCiP0gO3giO/frtZnY8kA5MFg2ruvFIrO0+38NZgBWE0/i/N2/stUzWA9PKWxzjB8rbnPOR5nAsxMgZR5T/Y29VrDfU9YvObg3vy6z+OIH15zucFRm+QWHKVdqZudXp7doycuYA1gs7Ob3nm2sQetOx2PaQU4HmcCLJ08DrYyWdI3NoSjHPFT91VUmI1ENbHgVADU0+94M8SH0OqfWuMBsJQNyWMcj/MC5o+y+hYSo2/y1vp/kCP+orsqQPOhqIPBglcMVPFrMoBVwFQXUBdz0cUd0fc4HmcFLGWwqWLRGmKABnLEX3qb8dZ9HQwxUPFrC1gBoN9xXUy/DZ5rBTgeZwKsfhtaA1apbMNzlCP+8ruMdKuDIQYqfg3Alku9QguOx5kAI7u5EVeqKIFWuMm+NuGIv/xejD5F5QdgyyXrsAXH42yAEVVhMtNDb3HeoThdh+aIX16TWiPFg4/bUveOdwHYovVZeYF4xvxlgoyAKY8sZMVnL17yDNzC9oC5z7S8x3jq3AlgbnZh+i2q5cD96OLyOR5nBqyL2+5HTI74/bhyXMWqCnA8BmCrqrukHUf8JSHxds8U4HgMwIRM5Igv1DXCZFKA4zEAEzKHI75Q1wiTSQGOxwBMyByO+EJdI0wmBTgeAzAhczjiC3WNMJkU4HgMwITM4Ygv1DXCZFKA4zEAEzKHI75Q1wiTSQGOxwBMyByO+EJdI0wmBTgeAzAhczjiC3WNMJkU4HgMwITM4Ygv1DXCZFKA4zEAEzKHI75Q1wiTSQGOx40AUx3gBxoc5RxoynIjwJ7PXxB+0hqopIM2aW3GoovyuOk/ACY0aACwccOlBgkAJgQLZ8QFYAAsVd1QwYSgBGAADIAJwZSqcAAMgAEwAIaNmBY5gDVYC/FSVanJMVQwVDBUsA4BBGAADIABMEwRW+QApogtxGsyHUy1RQVDBUMF6xBAAAbAABgAwxSxRQ5githCvNS0r8mxcVawO7S1MaWt/fFXp1W87i9g92/SmeLPZXt/Gtodm92kB2uEYxVxl7WRAcwktPv2T/N4jq7dX1eCAzDf9/4CVgvPeAxsDdj+Vf3VunG12L8CwPwkX+fzwQG2f2VKkxFUL2V6O8Ae0bXZlCZX7vRsjTSeAVACzIEBNi7z2gFmtDhz/REAq53trGuaXPY7KMAeXD9Hk42rtN9jQZuMeu0Ae0G6mi/TQ69lzZTRtHfr2bSOYZvEZoWdlvprvnCKmhgEi/V02Ofivso4RbveVesSpDrfBwRYX0fs5SIvEr/uvdWOG01UstdWsiK5fVjs9DKYaieOVdZ4qk0IiRn0/DVfCYa+B9d/AMcqfVm/Z+cGvSM5HMC02aG5qyUhH4Cu47etYO76ihE+BZpN8AqAWk8PjPi1nSXo2AGIsZ4RUHPvdRKuF/R8pb5MnP6tMeP7X/x6MIBpo4NRcPGNueTr86MUYO4eC9B8nXSSeyC56bVNfje9q9M3OS134LiPTAKwHWD1gKzWlzm/MjC46x/I4zAAi5LBJdTQH6UB03ro6uBNB1cCzE7ZPGD8NVa57rXQbPjAxiC4Nmqtl5pxNOvLDQBD9XoYgOmkSZk17CrWCWBzm8Cuiq0EmN0wcefUVAddeSrTxTRgW/sOpKpvdRUshMjEBWCp31a0x6QSaDVDhgeblD5hYprELqZWKwKWnApGoCUBsxWz6M9fgznYo0q2Sl/PgzjD89Z5MoAKFo+QwxXbie4e2wGmdKmpDv7xFQFzCV35EH//arGLV7djGO5ixpXHvA6ni/ZYXA29vtz1oIJ1XcF0gnhrimhUdck6xMf2gLnPtLzHOGlXBkwNXG5aVx9PV7FiraYAN7CkK5gbDB1kvo/L+jLnALCuARsRUPEg0A4wl7x4jHXt0+sBTBHHm0AAbLzeOsgB2BorJAADYKkJIP6yrxCUAAyAATAhmNyUwX8EYAAMgAGwnv13mGFBiTVYhwD51Sr1HBVsWLCkPFx2DIABMFSoDnMAgHUo7iqj27I2eH/YVQ6AATBUsA5zAIB1KO6y6qPEX9YG76OCpXYa9TEk0OLkgD6L9RnD4NJ5BVMd4AcaHOUcqK1ANW80+k2Omhg4TMT6BnoINywFOq9gw5Ij79VyxM97heitrQIcj1HB2qpuz+eIL9Q1wmRSgOMxABMyhyO+UNcIk0kBjscATMgcjvhCXSNMJgU4HgMwIXM44gt1jTCZFOB4DMCEzOGIL9Q1wmRSgOMxABMyhyO+UNcIk0kBjscATMgcjvhCXSNMJgU4HgMwIXM44gt1jTCZFOB4DMCEzOGIL9Q1wmRSgOMxABMyhyO+UNcIk0kBjscATMgcjvhCXSNMJgU4HmcG7JC2iz/VPKXJzmEmabrvhiN+cFV3dyn8miHzJ6+37wat8GKNCnA8zgiYhauA6indODulydk9erJG0aS65ogf9K0B26Qbj8ujT/Y2NXSArNRknc84HmcDzCTLLh34Cj3eo9nGlMaQQBzxfSkoARiRHYSKQSk4Ay8yK8DxOBtgBzupamWqGgAjAmCZaWF012vA6itYOC1i3HcvTuGIH1x4soKZAWi299Rraquav5ZNTrOj9e7GLh0k+/BC4+lCBTgeZ6tgRDVrsJFMfzjiB24mkj9Z9e/uUghcrCsR2al30E7HVxsn4xjQAu0yveB4nBEwpUI0qo4ELnVnHPGDvCgA8L4wb0UY9OzAq2IazIS2ZhYBwALdG7zgeJwPsMQIPSbDOeIH3ib0cZUo9XGGhsifJqopoA5oBrGgermOUn249/C4VAGOx5kAqzN9PLtkHPEDR+uSPzpuBqXwM0RzzAK2aGc2ihX0jxdLFeB4nAewBabr5PCmN0vvsqcNOOIHt1KX/Fo7O62r0RGABUp29oLjcR7A7NqrOm1BBSuyoQ4w/3gSMLer6KaI9ZqaaSXWYIXmDZ/0GDAiM8qG5ppj+KBZ++yD5Iy3QJVrsOqOodNwUqzB3Gdq03C3UcfHLqKTlvPYa8DKJPJ3ydyoy7ndfp3DET+4gwIAX5/E4OOgsxscalYQTBFd0Die2lVMQeza43GpAhyPM00Rl1774BtwxM9+0xqw8QxqufXjeAzAhFziiC/U9cphkh9cr3w2GnI8BmBCecMRX6jrShg1ZYx/v9NMI8M1cOVEHFioAMdjALZQ0tXf5Ii/evSGLeP1l16vYWrYUMVKc47HAKwiI+8AR3xeTzhrXQpwPAZgQm5xxBfqGmEyKcDxGIAJmcMRX6hrhMmkAMdjACZkDkd8oa4RJpMCHI8BmJA5HPGFukaYTApwPAZgQuZwxBfqGmEyKcDxGIAJmcMRX6hrhMmkAMdjACZkDkd8oa4RJpMCHI8bAaY6wA80OMo50JTlRoA9n78g/KQ1UEkHbdLajEUX5XHTfwBMaNAAYOOGSw0SAEwIFs6IC8AAWKq6oYIJQQnAABgAE4IpVeEAGAADYAAMGzEtcgBrsBbipapSk2OoYKhgqGAdAgjAABgAA2CYIrbIAUwRW4jXZDqYaosKhgqGCtYhgAAMgAEwAIYpYoscwBSxhXipaV+TY72sYPdv0pmNc3Ttfn11eXD9HE02rtJ+W+1W6KuJnn1s23/AtAnen4a+cmc0I6ooYE6n2U160CbxV0h6AFY/+MSQ9xuw/as02ZjS1r67oUd0bTalyUggkwRMJ/3s3NLqEydA5fURAmz/Sve51GPADExnrj8KK9YKCVBJmjYjeofnygFmtNrar9GsyT2soO9YKtjRBkwb7VevsIpVwGuSRD1pKwaYrvRmTWQqWYtpIgALB/SWudLfCrYEsDFME6UAC0biOt08cHT74ruao80Kr10xE7BT9Yld39VVsDBuanB0g6R99Ppa7dw7tFVct1mXpwdau5Tw2/rLCnc//vsSmzYJGPsL2NyIWRXQiuwLlrixIjl6/J4MYEaPpetUC15yTetvjHhJrzW0yej7UAXMJrQfx55XXlcEl/LF9nVmFsJo4ofHnieuw5wfraPcfQb5YXPGv775CwoGpo7ypMeAvaCU0PtXzpEyBBXMJqxOvLAKVQFwyTwlH5QSIG9b3gcsldRz54vXp27nxbDJqhM4Supg4LMwVK7JJX9x7oK1ZdR3LTS2Lx/42raCsPUaMGWGg0yNvG70zSFMkAiCgvtx21cwWzmC0bqEyU8mVy0qn2/FiecA21efhyWALDwpAavzIwm6r6XrK/GZW3DugnbPg5lOXM39qlnVqu66fY/aPu89YNUbXDCa+eYN4HlrwCwcbvCpPBYVwEFXrTJumlXA6Mf0z/f0DJJ/bhM3WM94n1suWtssACfoI6pSYU4YqHQVtNde3It3zc/ddXqDEQALBKoujEOh/dFqGM/bAhYkYaSVec8Dqi6Z46Qs2qXXLUrzuF92ohZ9Vf3SfTjAF7RDBYuMbwuFNtMJLxy77bU1Pb8dYNUpT9i/N7IrneqSVB/3NhSCdmnIYsDi1+F1VOEp3rd9V9dg8b1F9+L7HlS3+Dyv7/g+3TrPq2jFdfnxWz7v9RTxwfWrwe/EGSO9UbnlzXchaJOYrQDTieWBkdAiGIwCcBYkXtzOJqbbolf3VwUqDaLa+UtP12z/Lnb0u4/6uqNji3YRA0BdzAAce33BsdR9eLok9GzirWvbc8DUL5V68/mRVC5ffPe86aNJwnKjIXm+D2EMjksgm5AFCKl2Lmlt0lcBU4mZWIst86voywJQeF1zX8V1lDlRXLe7H/0Yx0tv1rjppcmxmj6DuM0B7DVgyaRpecN9itmqgo1Ihz55In0tAGyNiQrAmlcEaQC6jgfAAJjo7951nbBDiw/AABgA6zAHAFiH4i4bbTFFxBQRf5OjQwABGAADYAAMU8gWOYApYgvxlk0Bl72PCoYKhgrWIYAADIABMACGKWKLHMAUsYV4y6aAy95HBUMFQwXrEEAABsBaA6aSCD/Q4CjnQAqiRcdW/o7mRUHwHhSAAmkFAFhaFxyFAiIKADARGREECqQVAGBpXXAUCogo8P94MC6SwDPEtQAAAABJRU5ErkJggg==)

# Load the dataset and define the condition of cGAN model
X_train, y_train, X_test, y_test = load_dataset(dataset='fashion_mnist')
X_test = np.repeat(X_test, 3, 3)
print('ddd', X_test.shape)
num_classes = 10
class_labels = np.array([0,1,2,3,4,5,6,7,8,9] * 5)

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} colab_type="code" executionInfo={"elapsed": 14451, "status": "error", "timestamp": 1595151530297, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgTlAXb28EMHx7oSDz2IlYWnQTswhUuy-nr1hxDuA=s64", "userId": "16918877143175015576"}, "user_tz": -120} id="asL6PbIPZhAN" outputId="c2c35795-3d7f-4994-b12f-fe5249088c2a"
import os
physical_devices = tf.config.list_physical_devices('GPU') 
print(physical_devices)
   
img_w, img_h, num_channels = X_train[0].shape



print('\n\n---------------START CROSSVALIDATION----------------------------------\n\n')

start = time.time()

batch_size_list=[2, 32, 64, 128]

fid_score = crossvalidation(X_train=X_train, y_train=y_train, 
                            batch_size_list=batch_size_list,  
                            class_labels=class_labels, iterations=100000, 
                            save_name='crossvalidation_result_fashion_mnist_it_100000_3.txt', 
                            fashion_mnist=True)

end= time.time()

hours, rem = divmod(end-start, 3600)
minutes, seconds = divmod(rem, 60)

print('\n\n---------------END CROSSVALIDATION----------------------------------\n\n')



print("Total Execution Time: {:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes), seconds) )



# +
# LOAD RESULT CROSSVALIDATION PREVIOUSLY SAVED ON crossvalidation.txt

fjd = []
batch = []
with open('crossvalidation_result.txt', 'r') as f:
    results = json.loads(f.read())
    for r in results:
        batch.append(r[0])
        fjd.append(r[1])


# +
# PLOT RESULT CROSSVALIDATION


def auto_label(rects):
    """Attach a text label above each bar in *rects*, displaying its height."""
    for rect in rects:
        height = rect.get_height()
        plt.annotate('{:.1f}'.format(height),
                     xy=(rect.get_x() + rect.get_width() / 2, height),
                     xytext=(0, 3),  # 3 points vertical offset
                     textcoords="offset points",
                     ha='center', va='bottom',fontsize=20)

#plot resuls in a histogram
plt.figure(figsize=(15,10))

for b, f in zip(batch, fjd):
    curr_rects= plt.bar(b, f, width=15, label='batch size = '+str(b))
    auto_label(curr_rects)
plt.legend(fontsize=16)
plt.title('Kfold-Crossvalidation-100.000 iterations', size=20)
plt.ylabel('Frèchet Joint Distance (FJD)',size=20)
plt.xlabel('Batch Size',size=20)
plt.margins(x=0.4,y=0.3)
plt.xticks(batch)
plt.grid()

# + colab={} colab_type="code" id="j_xXGgbbY5EH"
#train model

img_w, img_h, num_channels = X_train[0].shape
cgan = CGAN(img_w, img_h, num_channels, num_classes)
discriminator = cgan.build_discriminator_model()
generator = cgan.build_generator()
cGAN= cgan.cgan(generator, discriminator)


img_w, img_h, num_channels = X_train[0].shape

best_batch_size_idx = np.argmin(fjd)

print('Best batch_size-FID: ', fjd[best_batch_size_idx], '-', batch[best_batch_size_idx])
best_batch_size = batch_size_list[best_batch_size_idx]

d_loss, g_loss, fid_list, tot_time = cgan.train(X_train, X_test, y_train, generator, discriminator, cGAN, class_labels, iterations=100000, batch_size=best_batch_size, sample_interval=500, k=1, save_model=True, verbose =True)

StarcoderdataJupyterScriptsDedupFiltered