<jupyter_start><jupyter_text>Imports and Settings<jupyter_code># basics
import os
import time
import numpy as np
import pandas as pd

# sklearn
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder, StandardScaler

# keras
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Dense, Dropout, Flatten
from keras.callbacks import TensorBoard, ModelCheckpoint

# tensorflow
import tensorflow as tf

# plotting
import matplotlib.pyplot as plt

# local
from data_loaders import load_mnist_data, load_iris_data, retrieve_predictions
from models import *
from utils import train_model_iteratively, get_model_weights, convert_weight_dict_to_dataframe

plt.style.use('ggplot')
%matplotlib inline
%load_ext autoreload
%autoreload 2
# suppress warnings
import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Working examples that use module functions  MNIST working example<jupyter_code>x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist = load_mnist_data()
pd.DataFrame(y_test_mnist).to_csv("y_test_mnist.csv", index=False)
baseline_mnist_model(name='mnist_test', num_classes=2).summary()
example_mnist_outdir = './data/example_mnist'
train_model_iteratively(baseline_model=baseline_mnist_model,
                        X_train=x_train_mnist, Y_train=y_train_mnist, 
                        X_test=x_test_mnist, Y_test=y_test_mnist,
                        outdir=example_mnist_outdir, epochs=4, epochs_to_save=None, 
                        batch_size=128, num_models=10)
mnist_preds = retrieve_mnist_preds(y_test=y_test_mnist, outdir=example_mnist_outdir)
mnist_weights = convert_weight_dict_to_dataframe(get_model_weights(example_mnist_outdir))<jupyter_output><empty_output><jupyter_text>IRIS working example<jupyter_code>x_train_iris, x_test_iris, y_train_iris, y_test_iris = load_iris_data()
pd.DataFrame(y_test_iris).to_csv("y_test_iris.csv", index=False)
baseline_iris_model(name='iris_test').summary()
example_iris_outdir = './data/example_iris'

# need this to suppress tf.function retracing warning that kept coming up
tf.compat.v1.logging.set_verbosity("ERROR")
train_model_iteratively(baseline_model=baseline_iris_model,
                        X_train=x_train_iris, Y_train=y_train_iris, 
                        X_test=x_test_iris, Y_test=y_test_iris,
                        outdir=example_iris_outdir, epochs=15, epochs_to_save=None, 
                        batch_size=5, num_models=2)<jupyter_output>X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)<jupyter_text>Model Runs  MNIST model runs  Baseline MNIST<jupyter_code>start = time.time()
train_model_iteratively(baseline_model=baseline_mnist_model,
                        X_train=x_train_mnist, Y_train=y_train_mnist, 
                        X_test=x_test_mnist, Y_test=y_test_mnist,
                        outdir="./data/mnist_baseline", epochs=4, epochs_to_save=None, 
                        batch_size=128, num_models=100)
print("train_model_iteratively took {} seconds\n".format(time.time() - start))<jupyter_output>X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 347.0760819911957 seconds<jupyter_text>Sweep epochs<jupyter_code>for num_epochs in [8, 12, 16]:
    outdir = "./data/mnist_epoch_{}".format(num_epochs)
    start = time.time()
    train_model_iteratively(baseline_model=baseline_mnist_model,
                            X_train=x_train_mnist, Y_train=y_train_mnist, 
                            X_test=x_test_mnist, Y_test=y_test_mnist,
                            outdir=outdir, epochs=num_epochs, epochs_to_save=None, 
                            batch_size=128, num_models=100)
    print("train_model_iteratively took {} seconds\n".format(time.time() - start))<jupyter_output>X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 637.4080867767334 seconds

X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 923.3448550701141 seconds

X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 1249.7377841472626 seconds<jupyter_text>Sweep batch size<jupyter_code>for batch_size in [32, 64, 256]:
    outdir = "./data/mnist_batch_{}".format(batch_size)
    start = time.time()
    train_model_iteratively(baseline_model=baseline_mnist_model,
                            X_train=x_train_mnist, Y_train=y_train_mnist, 
                            X_test=x_test_mnist, Y_test=y_test_mnist,
                            outdir=outdir, epochs=4, epochs_to_save=None, 
                            batch_size=batch_size, num_models=100)
    print("train_model_iteratively took {} seconds\n".format(time.time() - start))<jupyter_output>X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 821.0129718780518 seconds

X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 459.6735169887543 seconds

X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 303.78163290023804 seconds<jupyter_text>Sweep layers<jupyter_code>for idx, model in enumerate([mnist_2_layers, mnist_3_layers, mnist_4_layers]):
    outdir = "./data/mnist_layers_{}".format(idx+2)
    start = time.time()
    train_model_iteratively(baseline_model=model,
                            X_train=x_train_mnist, Y_train=y_train_mnist, 
                            X_test=x_test_mnist, Y_test=y_test_mnist,
                            outdir=outdir, epochs=4, epochs_to_save=None, 
                            batch_size=128, num_models=100)
    print("train_model_iteratively took {} seconds\n".format(time.time() - start))<jupyter_output>X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 559.6651039123535 seconds

X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 788.8657438755035 seconds

X_train: (12665, 28, 28, 1), Y_train: (12665, 2)
X_test: (100, 28, 28, 1), Y_test: (100, 2)
train_model_iteratively took 932.9938669204712 seconds<jupyter_text>IRIS model runs  Baseline IRIS<jupyter_code>start = time.time()
train_model_iteratively(baseline_model=baseline_iris_model,
                        X_train=x_train_iris, Y_train=y_train_iris, 
                        X_test=x_test_iris, Y_test=y_test_iris,
                        outdir="./data/iris_baseline", epochs=150, epochs_to_save=None, 
                        batch_size=5, num_models=100)
print("train_model_iteratively took {} seconds\n".format(time.time() - start))<jupyter_output>X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 774.9198100566864 seconds<jupyter_text>Sweep epochs<jupyter_code>for num_epochs in [100, 300, 450]:
    outdir = "./data/iris_epoch_{}".format(num_epochs)
    start = time.time()
    train_model_iteratively(baseline_model=baseline_iris_model,
                            X_train=x_train_iris, Y_train=y_train_iris, 
                            X_test=x_test_iris, Y_test=y_test_iris,
                            outdir=outdir, epochs=num_epochs, epochs_to_save=None, 
                            batch_size=5, num_models=100)
    print("train_model_iteratively took {} seconds\n".format(time.time() - start))<jupyter_output>X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 507.4716958999634 seconds

X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 1513.5248148441315 seconds

X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 2045.6344740390778 seconds<jupyter_text>Sweep batch size:<jupyter_code>for batch_size in [10, 20, 40]:
    outdir = "./data/iris_batch_{}".format(batch_size)
    start = time.time()
    train_model_iteratively(baseline_model=baseline_iris_model,
                            X_train=x_train_iris, Y_train=y_train_iris, 
                            X_test=x_test_iris, Y_test=y_test_iris,
                            outdir=outdir, epochs=150, epochs_to_save=None, 
                            batch_size=batch_size, num_models=100)
    print("train_model_iteratively took {} seconds\n".format(time.time() - start))<jupyter_output>X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 668.1161737442017 seconds

X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 741.2695188522339 seconds

X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 589.3309698104858 seconds<jupyter_text>Sweep layers:<jupyter_code>for idx, model in enumerate([iris_2_layers, iris_3_layers, iris_4_layers]):
    outdir = "./data/iris_layers_{}".format(idx+2)
    start = time.time()
    train_model_iteratively(baseline_model=model,
                            X_train=x_train_iris, Y_train=y_train_iris, 
                            X_test=x_test_iris, Y_test=y_test_iris,
                            outdir=outdir, epochs=150, epochs_to_save=None, 
                            batch_size=5, num_models=100)
    print("train_model_iteratively took {} seconds\n".format(time.time() - start))<jupyter_output>X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 726.9213287830353 seconds

X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 773.2967009544373 seconds

X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 769.7694590091705 seconds<jupyter_text>Sweep nodes:<jupyter_code>for idx, model in enumerate([iris_3_nodes, iris_4_nodes, iris_5_nodes, iris_6_nodes]):
    outdir = "./data/iris_nodes_{}".format(idx+3)
    start = time.time()
    train_model_iteratively(baseline_model=model,
                            X_train=x_train_iris, Y_train=y_train_iris, 
                            X_test=x_test_iris, Y_test=y_test_iris,
                            outdir=outdir, epochs=150, epochs_to_save=None, 
                            batch_size=5, num_models=100)
    print("train_model_iteratively took {} seconds\n".format(time.time() - start))<jupyter_output>X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 720.6743988990784 seconds

X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 726.2040410041809 seconds

X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 729.7382500171661 seconds

X_train: (75, 2), Y_train: (75, 3)
X_test: (75, 2), Y_test: (75, 3)
train_model_iteratively took 765.2918190956116 seconds
StarcoderdataJupyterStructuredCleanDedup