# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Compare performance of ImageDataGenerator and tf.data
# A quick experiment to evaluate how tf.data stands up to ImageDataGenerator. Experiment was conducted 5 times with max deviation from these results ~12%

# +
import tensorflow as tf
import tensorflow.keras as keras
import numpy as np

DATA_DIR = 'a folder with 320 jpeg images 299x299x3'
keras.__version__,tf.__version__
# -

iv3 = keras.applications.inception_v3.InceptionV3(include_top=False, weights='imagenet',input_shape=(299, 299, 3))


syntetic_data = np.random.randint(0,255,size=(6400,299,299,3))

# ## In-memory data as a baseline 

# %%time
iv3.predict(syntetic_data,batch_size=64).shape

# ## Using ImageDataGenerator

# +
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import glob
import pandas as pd

files = glob.glob(DATA_DIR+'/*.JPG')
df = pd.DataFrame({'filename': files})

gen = ImageDataGenerator() \
        .flow_from_dataframe(df,class_mode=None,batch_size=64,target_size=(299,299))
# -

# %%time
for i in range(100):
    #nxt = next(gen)
    nxt = keras.applications.inception_v3.preprocess_input(next(gen)) # needed for tensorflow backend
    #print('batch {} : {} {}'.format(i,nxt.shape, nxt.dtype))

iv3.compile(optimizer='adam',loss='mse') # why would that be required for predict_generator? -> RuntimeError: You must compile your model before using it.

# %%time
out_generator = iv3.predict_generator(gen,steps=100)
print(out_generator.shape)

# ## Using tf.data pipeline

# +
dataset = tf.data.Dataset.from_tensor_slices(files)
def _load_images(filename):
    image = tf.read_file(filename)
    image = tf.image.decode_jpeg(image,channels=3)
    image = tf.image.resize_images(image, [299, 299])
    image = keras.applications.inception_v3.preprocess_input(image)
    return image

dataset = dataset.map(_load_images)
dataset = dataset.batch(64).repeat()

iterator = dataset.make_one_shot_iterator()
image = iterator.get_next()
# -

# %%time
with tf.Session() as sess:
    for i in range(100):
        out = sess.run(image)
        #print('batch {} : {} {}'.format(i,out.shape, out.dtype))

# %%time
out_tfdata = iv3.predict(image,steps=100)
print(out_tfdata.shape)

# ## Some observations
# * tf.data and ImageDataGenerator have similar performance on a singe machine. Maybe for distrubuted training it will be worth it to use tf.data (and for TPU it is required).
# * Observed CPU (i7-7820HQ) load was ~35% while GPU (NVidia Quadro M2200) load was 100%. 
# * Data loading (from SSD) does not appear as a bottleneck for big models (e.g. Inception v3). However, smaller models or faster/more GPUs may require faster disks/RAID ...



StarcoderdataJupyterScriptsDedupFiltered