# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] colab_type="text" id="rl85J9Iqh4yQ"
# # Dog Breeds analysis
#
# Sample pandas examples:
#
# https://github.com/rasbt/pattern_classification/blob/master/data_viz/matplotlib_viz_gallery.ipynb
# https://github.com/rasbt/pattern_classification/blob/master/data_viz/matplotlib_viz_gallery.ipynb
#

# + [markdown] colab_type="text" id="CwvFftZiqrRE"
# ### Processing for using Google Drive and normal includes
#
# The notebook uses TensorFlow 2.x.  (Eager execution is enabled by default and we use the newer versions of tf.Data.)
#
# I use Notebooks with Colab and on my local workstation, so I need to seperate some logic to make it easier to run in both locations.
#
# I was going to delete and just make Colab version, but that is not "real world."  You usually have multiple environments and I'm showing you how I accommodate different environments, you might need something different...
#
#

# + colab_type="code" id="xjJySeIXh_md" colab={}
#"""
# Google Collab specific stuff....
from google.colab import drive
drive.mount('/content/drive')

import os
# !ls "/content/drive/My Drive"

USING_COLLAB = True
# Force to use 2.x version of Tensorflow
# %tensorflow_version 2.x
#"""

# + colab_type="code" id="6w1an-l5h4yT" colab={}
# Setup sys.path to find MachineLearning lib directory

# Check if "USING_COLLAB" is defined, if yes, then we are using Colab, otherwise set to False
try: USING_COLLAB
except NameError: USING_COLLAB = False

# %load_ext autoreload
# %autoreload 2

# set path env var
import sys
if "MachineLearning" in sys.path[0]:
    pass
else:
    print(sys.path)
    if USING_COLLAB:
        sys.path.insert(0, '/content/drive/My Drive/GitHub/MachineLearning/lib')  ###### CHANGE FOR SPECIFIC ENVIRONMENT
    else:
        sys.path.insert(0, '/Users/john/Documents/GitHub/MachineLearning/lib')  ###### CHANGE FOR SPECIFIC ENVIRONMENT
    
    print(sys.path)

# + colab_type="code" id="csTt9CUvh4yZ" colab={}
# Normal includes...

from __future__ import absolute_import, division, print_function, unicode_literals

import os, sys, random, warnings, time, copy, csv
import numpy as np 

import IPython.display as display
from PIL import Image
import pandas as pd
import matplotlib.pyplot as plt
# %matplotlib inline

import tensorflow as tf
print(tf.__version__)

# This allows the runtime to decide how best to optimize CPU/GPU usage
AUTOTUNE = tf.data.experimental.AUTOTUNE

from TrainingUtils import *

#warnings.filterwarnings("ignore", category=DeprecationWarning)
#warnings.filterwarnings("ignore", category=UserWarning)
warnings.filterwarnings("ignore", "(Possibly )?corrupt EXIF data", UserWarning)

# + [markdown] colab_type="text" id="aplx71Xjh4yg"
# ## General Setup
#
# - Create a dictionary wrapped by a class for global values.  This is how I manage global vars in my notebooks.
#
#

# + colab_type="code" id="PMdwqph-h4yd" colab={}
# Set root directory path to data
if USING_COLLAB:
    ROOT_PATH = "/content/drive/My Drive/GitHub/MachineLearning/9-LibTest/Data"  ###### CHANGE FOR SPECIFIC ENVIRONMENT
else:
    ROOT_PATH = "/Users/john/Documents/GitHub/MachineLearning/9-LibTest/Data"  ###### CHANGE FOR SPECIFIC ENVIRONMENT
        
# Establish global dictionary
parms = GlobalParms(ROOT_PATH=ROOT_PATH,
                    TRAIN_DIR="CatDogLabeledVerySmall", 
                    NUM_CLASSES=120,
                    IMAGE_ROWS=224,
                    IMAGE_COLS=224,
                    IMAGE_CHANNELS=3,
                    BATCH_SIZE=1,  # must be one if you want to see different image sizes
                    IMAGE_EXT=".jpg")

parms.print_contents()


# + colab_type="code" id="vWyqw8Yw5rF7" colab={}
# Simple helper method to display batches of images with labels....        
def show_batch(image_batch, label_batch, number_to_show=25, r=5, c=5, print_shape=False):
    show_number = min(number_to_show, parms.BATCH_SIZE)

    if show_number < 8: #if small number, then change row, col and figure size
        if parms.IMAGE_COLS > 64 or parms.IMAGE_ROWS > 64:
            plt.figure(figsize=(25,25)) 
        else:
            plt.figure(figsize=(10,10))  
        r = 4
        c = 2 
    else:
        plt.figure(figsize=(10,10))  

    #if show_number == 1:
    #  image_batch = np.expand_dims(image_batch, axis=0)
    #  label_batch = np.expand_dims(label_batch, axis=0)

    for n in range(show_number):
        if print_shape:
            print("Image shape: {}  Max: {}  Min: {}".format(image_batch[n].shape, np.max(image_batch[n]), np.min(image_batch[n])))

        ax = plt.subplot(r,c,n+1)
        plt.imshow(tf.keras.preprocessing.image.array_to_img(image_batch[n]))
        plt.title(parms.CLASS_NAMES[np.argmax(label_batch[n])])
        plt.axis('off')


# + [markdown] colab_type="text" id="0wJ5iS1Tu3Jq"
# ### Create dataset and normal mappings
#
# Pipeline Flow:
#
# create dataset -> map "process_path" -> repeat forever -> batch
#
# The mappings open and read an image.  These next cells should be changed based on your specific needs.
#

# + colab_type="code" id="uaK22hoX5rFr" colab={}
# Download dataset to local VM

import tensorflow_datasets as tfds
datasets, info = tfds.load(name='stanford_dogs', with_info=True, as_supervised=True)


# + colab_type="code" id="a1UcgQXIoWsS" colab={}
# Set Class names...
parms.set_class_names(['Chihuahua', 'Japanese_spaniel', 'Maltese_dog', 'Pekinese', 'Shih-Tzu', 'Blenheim_spaniel', 'papillon', 'toy_terrier', 'Rhodesian_ridgeback', 'Afghan_hound', 'basset', 'beagle', 'bloodhound', 'bluetick', 'black-tan_coonhound', 'Walker_hound', 'English_foxhound', 'redbone', 'borzoi', 'Irish_wolfhound', 'Italian_greyhound', 'whippet', 'Ibizan_hound', 'Norwegian_elkhound', 'otterhound', 'Saluki', 'Scottish_deerhound', 'Weimaraner', 'Staffordshire_bullterrier', 'American_Staffordshire_terrier', 'Bedlington_terrier', 'Border_terrier', 'Kerry_blue_terrier', 'Irish_terrier', 'Norfolk_terrier', 'Norwich_terrier', 'Yorkshire_terrier', 'wire-haired_fox_terrier', 'Lakeland_terrier', 'Sealyham_terrier', 'Airedale', 'cairn', 'Australian_terrier', 'Dandie_Dinmont', 'Boston_bull', 'miniature_schnauzer', 'giant_schnauzer', 'standard_schnauzer', 'Scotch_terrier', 'Tibetan_terrier', 'silky_terrier', 'soft-coated_wheaten_terrier', 'West_Highland_white_terrier', 'Lhasa', 'flat-coated_retriever', 'curly-coated_retriever', 'golden_retriever', 'Labrador_retriever', 'Chesapeake_Bay_retriever', 'German_short-haired_pointer', 'vizsla', 'English_setter', 'Irish_setter', 'Gordon_setter', 'Brittany_spaniel', 'clumber', 'English_springer', 'Welsh_springer_spaniel', 'cocker_spaniel', 'Sussex_spaniel', 'Irish_water_spaniel', 'kuvasz', 'schipperke', 'groenendael', 'malinois', 'briard', 'kelpie', 'komondor', 'Old_English_sheepdog', 'Shetland_sheepdog', 'collie', 'Border_collie', 'Bouvier_des_Flandres', 'Rottweiler', 'German_shepherd', 'Doberman', 'miniature_pinscher', 'Greater_Swiss_Mountain_dog', 'Bernese_mountain_dog', 'Appenzeller', 'EntleBucher', 'boxer', 'bull_mastiff', 'Tibetan_mastiff', 'French_bulldog', 'Great_Dane', 'Saint_Bernard', 'Eskimo_dog', 'malamute', 'Siberian_husky', 'affenpinscher', 'basenji', 'pug', 'Leonberg', 'Newfoundland', 'Great_Pyrenees', 'Samoyed', 'Pomeranian', 'chow', 'keeshond', 'Brabancon_griffon', 'Pembroke', 'Cardigan', 'toy_poodle', 'miniature_poodle', 'standard_poodle', 'Mexican_hairless', 'dingo', 'dhole', 'African_hunting_dog'])

print("Classes: ", parms.NUM_CLASSES, 
      "   Labels: ", len(parms.CLASS_NAMES), 
      "  ", parms.CLASS_NAMES)


# + [markdown] colab_type="text" id="USATdbIyi8Ca"
# ### Create dataset from list of images and apply mappings

# + colab_type="code" id="M-LXAs3e5rFx" colab={}
# Create Dataset from list of images
full_dataset = datasets['train']
full_len = info.splits['train'].num_examples

#full_dataset = datasets['test']  
#full_len = info.splits['test'].num_examples

# Verify image paths were loaded and save one path for later in "some_image"
for f in full_dataset.take(2):
    some_image = f[0]
    print(f[1])
    
#print("Some Image: ", some_image)

# + colab_type="code" id="-TZM_MFp5rGA" colab={}
# Repeat forever
full_dataset = full_dataset.repeat()

# set the batch size
full_dataset = full_dataset.batch(parms.BATCH_SIZE)


# + colab_type="code" id="T1_Q_XaoBtZt" colab={}
#create simple iterator
ds_iter = iter(full_dataset)


# + colab_type="code" id="UHmOZtlv5rGG" colab={}
# Show the images, execute this cell multiple times to see the images
# Execute at least 4 times if random is applied

image_batch, label_batch = next(ds_iter)
show_batch(image_batch.numpy(), label_batch.numpy())


# + [markdown] colab_type="text" id="ZQ7y2k0pjHQ4"
# ### Collect image information
#
# This will loop over each image and collect information to be used to create a Pandas dataframe.  The dataframe will then be used to report information.  You can also save the dataframe for future analysis.
#
# This is where you can also customize what information is collected.
#
# The size of the image is not changed, but you can change so every image is exactly like how it will be used for training.  I've found that looking at the raw image information is more helpful than looking at images that have been resized.
#

# + colab_type="code" id="BCM_Xd4irwy_" colab={}
# Collect various information about an image
def dataset_analysis(ds_iter, steps, test=False):
    if test == True:
        steps = 4

    image_info = []

    for i in tqdm(range(int(steps))):
        image_batch, label_batch = next(ds_iter)
        #show_batch(image_batch.numpy(), label_batch.numpy())

        for j in range(parms.BATCH_SIZE):
            image = image_batch[j].numpy()
            label = label_batch[j].numpy()
            #label = np.argmax(label)
            r = image.shape[0]
            c = image.shape[1]
            d = 0
            mean0=0
            mean1=0
            mean2=0
            if parms.IMAGE_CHANNELS == 3:
                d = image.shape[2]
                mean0 = np.mean(image[:,:,0])
                mean1 = np.mean(image[:,:,1])
                mean2 = np.mean(image[:,:,2])
            image_info.append([label, r, c, d, np.mean(image), np.std(image), mean0, mean1, mean2])

            if test:
                print(image_info[-1])
                
    return image_info


# + colab_type="code" id="JJIlgXSStQIY" colab={}
# Build image_info list

ds_iter = iter(full_dataset)

steps = np.ceil(full_len // parms.BATCH_SIZE)

image_info = dataset_analysis(ds_iter, steps=steps, test=False)

# + colab_type="code" id="Pas-QSw6p8gA" colab={}
# Build pandas dataframe
image_info_df = pd.DataFrame(image_info, columns =['label', 'row','col', 'dim', 'mean', 'std', "chmean0", "chmean1", "chmean2"])
print(image_info_df.describe())
image_info_df.head()

# + colab_type="code" id="YO68MUsjp8kX" colab={}
#https://jamesrledoux.com/code/group-by-aggregate-pandas
image_info_df.groupby('label').agg({'mean': ['count', 'mean', 'min', 'max'], 'std': ['mean', 'min', 'max'], 'row': ['mean', 'min', 'max'],'col': ['mean', 'min', 'max'], 'chmean0':['mean'],'chmean1':['mean'],'chmean2':['mean'] })


# + colab_type="code" id="JBVoJW-Xp8o3" colab={}
image_info_df.agg({'mean': ['mean', 'min', 'max'], 'std': ['mean', 'min', 'max'], 'row': ['mean', 'min', 'max'],'col': ['mean', 'min', 'max'] })


# + colab_type="code" id="_3lyzlN6qFcc" colab={}
image_mean = image_info_df["mean"]
print("Mean: ", np.mean(image_mean), "  STD: ", np.std(image_mean))

# + colab_type="code" id="E6_tIxRQqIOn" colab={}
plt.rcParams['figure.figsize'] = [20, 10]
image_info_df["label"].value_counts().plot.bar()

# + colab_type="code" id="GL1wjemvqLFy" colab={}
plt.rcParams['figure.figsize'] = [20, 10]
image_info_df["label"].value_counts().plot.pie()

# + colab_type="code" id="AW_A3EyHqQsY" colab={}
image_info_df.hist(column='mean')

# + colab_type="code" id="p9uHkKRqqVnv" colab={}
image_info_df.plot.scatter(x='row', y='col', color='Blue', label='Row-Col')

# + colab_type="code" id="7FlM5KWjqZSd" colab={}
# Save results
result_path = os.path.join(parms.ROOT_PATH, "image-info.pkl")
image_info_df.to_pickle(result_path)  

# + colab_type="code" id="wbxDcF1yqcjY" colab={}
# open and read saved file
image_info_df = pd.read_pickle(result_path)
image_info_df.head()

# + [markdown] colab_type="text" id="goQF3VdzJc9z"
# ### Final Thoughts.....
#
# Classes are balanced - like they should be :).  
#
# Image size is mainly clustered less than 600, so resizing to 224 should be ok - won't lose too much detail.
#
# Mean and STD are consistent across classes.
#
# Should be able to apply normal augmentation techniques. 

StarcoderdataJupyterScriptsDedupFiltered