<jupyter_start><jupyter_text>TensorFlow2: Training Loop. ![gradient](../images/gradient_descent.png) Although Keras is suitable for the vast majority of use cases, in the following scenarios, it may make sense to forgo `model.fit()` to manually define a training loop:- Maintaining legacy code and retraining old models.- Custom batch/ epoch operations like gradients and backpropagation. Even then, PyTorch may be a better fit for customization.> Disclaimer; This notebook demonstrates how to manually define a training loop for queued tuning of a binary classification model. However, it is only included to prove that AIQC technically supports TensorFlow out-of-the-box with `analysis_type='keras'`, and to demonstrate how expert practicioners to do continue to use their favorite tools. We neither claim to be experts on the inner-workings of TensorFlow, nor do we intend to troubleshoot advanced methodologies for users that are in over their heads.Reference this repository for more TensorFlow cookbooks: > https://github.com/IvanBongiorni/TensorFlow2.0_Notebooks<jupyter_code>import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout

from sklearn.preprocessing import LabelBinarizer, PowerTransformer

import aiqc
from aiqc import datum<jupyter_output><empty_output><jupyter_text>---  Example Data Reference [Example Datasets](example_datasets.ipynb) for more information.<jupyter_code>df = datum.to_pandas('sonar.csv')
df.head()<jupyter_output><empty_output><jupyter_text>---  a) High-Level API Reference [High-Level API Docs](api_high_level.ipynb) for more information including how to work with non-tabular data.<jupyter_code>splitset = aiqc.Pipeline.Tabular.make(
    df_or_path = df
    , dtype = None

    , feature_cols_excluded = 'object'
    , feature_interpolaters = None
    , feature_window = None
    , feature_encoders = dict(
        sklearn_preprocess = PowerTransformer(method='yeo-johnson', copy=False)
        , dtypes = ['float64']
    )
    , feature_reshape_indices = None

    , label_column = 'object'
    , label_interpolater = None
    , label_encoder = dict(sklearn_preprocess = LabelBinarizer(sparse_output=False))

    , size_test = 0.12
    , size_validation = 0.22
    , fold_count = None
    , bin_count = None
)
def fn_build(features_shape, label_shape, **hp):
    model = Sequential(name='Sonar')
    model.add(Dense(hp['neuron_count'], activation='relu', kernel_initializer='he_uniform'))
    model.add(Dropout(0.30))
    model.add(Dense(hp['neuron_count'], activation='relu', kernel_initializer='he_uniform'))
    model.add(Dropout(0.30))
    model.add(Dense(hp['neuron_count'], activation='relu', kernel_initializer='he_uniform'))
    model.add(Dense(units=label_shape[0], activation='sigmoid', kernel_initializer='glorot_uniform'))
    return model
def fn_lose(**hp):
	loser = tf.losses.BinaryCrossentropy()
	return loser
def fn_optimize(**hp):
	optimizer = tf.optimizers.Adamax()
	return optimizer
def fn_train(model, loser, optimizer, samples_train, samples_evaluate, **hp):
    batched_train_features, batched_train_labels = aiqc.tf_batcher(
        features = samples_train['features']
        , labels = samples_train['labels']
        , batch_size = 5
    )
    
    # Still necessary for saving entire model.
    model.compile(loss=loser, optimizer=optimizer)
    
    ## --- Metrics ---
    acc = tf.metrics.BinaryAccuracy()
    # Mirrors `keras.model.History.history` object.
    history = {
        'loss':list(), 'accuracy': list(), 
        'val_loss':list(), 'val_accuracy':list()
    }

    ## --- Training loop ---
    for epoch in range(hp['epochs']):
        # --- Batch training ---
        for i, batch in enumerate(batched_train_features):      
            
            with tf.GradientTape() as tape:
                batch_loss = loser(
                    batched_train_labels[i],
                    model(batched_train_features[i])
                )
            # Update weights based on the gradient of the loss function.
            gradients = tape.gradient(batch_loss, model.trainable_variables)            
            optimizer.apply_gradients(zip(gradients, model.trainable_variables))

        ## --- Epoch metrics ---
        # Overall performance on training data.
        train_probability = model.predict(samples_train['features'])
        train_loss = loser(samples_train['labels'], train_probability)
        train_acc = acc(samples_train['labels'], train_probability)
        history['loss'].append(float(train_loss))
        history['accuracy'].append(float(train_acc))
        # Performance on evaluation data.
        eval_probability = model.predict(samples_evaluate['features'])
        eval_loss = loser(samples_evaluate['labels'], eval_probability)
        eval_acc = acc(samples_evaluate['labels'], eval_probability)
        history['val_loss'].append(float(eval_loss))
        history['val_accuracy'].append(float(eval_acc))
    # Attach history to the model so we can return a single object.
    model.history.history = history 
    return model
hyperparameters = {
    "neuron_count": [25, 50]
    , "epochs": [75, 150]
}
queue = aiqc.Experiment.make(
    library = "keras"
    , analysis_type = "classification_binary"
    , fn_build = fn_build
    , fn_train = fn_train
    , fn_lose = fn_lose
    , fn_optimize = fn_optimize
    , splitset_id = splitset.id
    , repeat_count = 1
    , hide_test = False
    , hyperparameters = hyperparameters

    , fn_predict = None #automated
    , foldset_id = None
)
queue.run_jobs()<jupyter_output>🔮 Training Models 🔮: 100%|██████████████████████████████████████████| 4/4 [02:18<00:00, 34.72s/it]<jupyter_text>For more information on visualization of performance metrics, reference the [Visualization & Metrics](visualization.html) documentation. ---  b) Low-Level API Reference [Low-Level API Docs](api_high_level.ipynb) for more information including how to work with non-tabular data and defining optimizers.<jupyter_code>dataset = aiqc.Dataset.Tabular.from_pandas(df)
label_column = 'object'
label = dataset.make_label(columns=[label_column])
labelcoder = label.make_labelcoder(
    sklearn_preprocess = LabelBinarizer(sparse_output=False)
)
feature = dataset.make_feature(exclude_columns=[label_column])
encoderset = feature.make_encoderset()
featurecoder_0 = encoderset.make_featurecoder(
    sklearn_preprocess = PowerTransformer(method='yeo-johnson', copy=False)
    , dtypes = ['float64']
)
splitset = aiqc.Splitset.make(
    feature_ids = [feature.id]
    , label_id = label.id
    , size_test = 0.22
    , size_validation = 0.12
)
def fn_build(features_shape, label_shape, **hp):
    model = Sequential(name='Sonar')
    model.add(Dense(hp['neuron_count'], activation='relu', kernel_initializer='he_uniform'))
    model.add(Dropout(0.30))
    model.add(Dense(hp['neuron_count'], activation='relu', kernel_initializer='he_uniform'))
    model.add(Dropout(0.30))
    model.add(Dense(hp['neuron_count'], activation='relu', kernel_initializer='he_uniform'))
    model.add(Dense(units=label_shape[0], activation='sigmoid', kernel_initializer='glorot_uniform'))
    return model
def fn_lose(**hp):
	loser = tf.losses.BinaryCrossentropy()
	return loser
def fn_optimize(**hp):
	optimizer = tf.optimizers.Adamax()
	return optimizer
def fn_train(model, loser, optimizer, samples_train, samples_evaluate, **hp):
    batched_train_features, batched_train_labels = aiqc.tf_batcher(
        features = samples_train['features']
        , labels = samples_train['labels']
        , batch_size = 5
    )
    
    # Still necessary for saving entire model.
    model.compile(loss=loser, optimizer=optimizer)
    
    ## --- Metrics ---
    acc = tf.metrics.BinaryAccuracy()
    # Mirrors `keras.model.History.history` object.
    history = {
        'loss':list(), 'accuracy': list(), 
        'val_loss':list(), 'val_accuracy':list()
    }

    ## --- Training loop ---
    for epoch in range(hp['epochs']):
        # --- Batch training ---
        for i, batch in enumerate(batched_train_features):      
            
            with tf.GradientTape() as tape:
                batch_loss = loser(
                    batched_train_labels[i],
                    model(batched_train_features[i])
                )
            # Update weights based on the gradient of the loss function.
            gradients = tape.gradient(batch_loss, model.trainable_variables)            
            optimizer.apply_gradients(zip(gradients, model.trainable_variables))

        ## --- Epoch metrics ---
        # Overall performance on training data.
        train_probability = model.predict(samples_train['features'])
        train_loss = loser(samples_train['labels'], train_probability)
        train_acc = acc(samples_train['labels'], train_probability)
        history['loss'].append(float(train_loss))
        history['accuracy'].append(float(train_acc))
        # Performance on evaluation data.
        eval_probability = model.predict(samples_evaluate['features'])
        eval_loss = loser(samples_evaluate['labels'], eval_probability)
        eval_acc = acc(samples_evaluate['labels'], eval_probability)
        history['val_loss'].append(float(eval_loss))
        history['val_accuracy'].append(float(eval_acc))
    # Attach history to the model so we can return a single object.
    model.history.history = history 
    return model
algorithm = aiqc.Algorithm.make(
    library = "keras"
    , analysis_type = "classification_binary"
    , fn_build = fn_build
    , fn_train = fn_train
    , fn_lose = fn_lose
    , fn_optimize = fn_optimize
)
hyperparameters = {
    "neuron_count": [25, 50]
    , "epochs": [75, 150]
}
hyperparameters = {
    "neuron_count": [25, 50]
    , "epochs": [75, 150]
}
hyperparamset = algorithm.make_hyperparamset(
    hyperparameters = hyperparameters
)
queue = algorithm.make_queue(
    splitset_id = splitset.id
    , hyperparamset_id = hyperparamset.id
    , repeat_count = 2
)
queue.run_jobs()<jupyter_output>🔮 Training Models 🔮: 100%|██████████████████████████████████████████| 8/8 [04:25<00:00, 33.17s/it]
StarcoderdataJupyterStructuredCleanDedup