# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="1DtGV4xh7SND" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1642108631192, "user_tz": -330, "elapsed": 20662, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="56743038-8cb5-4278-b5a7-491328de576e"
from google.colab import drive
drive.mount('/content/drive')

# + id="BZ0g9z9E7gso" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1642108631872, "user_tz": -330, "elapsed": 682, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="65901ce5-c721-4aff-fc4a-ef3f9ef54ced"
# %cd /content/drive/My Drive/semeval_2022/maMi

# + colab={"base_uri": "https://localhost:8080/"} id="KrO9sy8rpryV" executionInfo={"status": "ok", "timestamp": 1642108635115, "user_tz": -330, "elapsed": 3245, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="520bcd5e-6916-4429-a7da-8baf75f470ac"
# !pip install tensorflow_addons

# + id="XEDCSKN17guQ" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1642108637394, "user_tz": -330, "elapsed": 2281, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="a6e006e0-c6b5-4275-f30d-b7c336211767"
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow_addons as tfa
import glob, warnings
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix, classification_report
import seaborn as sns

warnings.filterwarnings('ignore')
print('TensorFlow Version ' + tf.__version__)

# + id="zwFbHyP77gyJ"
IMAGE_SIZE = 224
BATCH_SIZE = 16
EPOCHS = 7
img_folder='./TRAINING/'
train_df=pd.read_csv('train.csv',sep='\t')
val_df=pd.read_csv('test.csv',sep='\t')

# + id="I3vVLqD5Q2W_" executionInfo={"status": "ok", "timestamp": 1642108644084, "user_tz": -330, "elapsed": 383, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}}
IMAGE_SIZE = 224
BATCH_SIZE = 16
img_folder='./test/'
test_df=pd.read_csv('./test/Test.csv',sep='\t')


# + id="xxIh3AdA7g0f"
def data_augment(image):
    p_spatial = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
    p_rotate = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
    p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
    p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
    p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
    
    # Flips
    image = tf.image.random_flip_left_right(image)
    image = tf.image.random_flip_up_down(image)
    
    if p_spatial > .75:
        image = tf.image.transpose(image)
        
    # Rotates
    if p_rotate > .75:
        image = tf.image.rot90(image, k = 3) # rotate 270º
    elif p_rotate > .5:
        image = tf.image.rot90(image, k = 2) # rotate 180º
    elif p_rotate > .25:
        image = tf.image.rot90(image, k = 1) # rotate 90º
        
    # Pixel-level transforms
    if p_pixel_1 >= .4:
        image = tf.image.random_saturation(image, lower = .7, upper = 1.3)
    if p_pixel_2 >= .4:
        image = tf.image.random_contrast(image, lower = .8, upper = 1.2)
    if p_pixel_3 >= .4:
        image = tf.image.random_brightness(image, max_delta = .1)
        
    return image


# + id="v7WgiogjRKv7"
#seeee class mode

# + id="1HP04i6q7g4S" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1640348055665, "user_tz": -330, "elapsed": 53484, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="3950aff6-dbf4-45d1-f77d-1c06975262ec"
datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale = 1./255,
                                                          samplewise_center = True,
                                                          samplewise_std_normalization = True,
                                                          validation_split = 0.2,
                                                          preprocessing_function = data_augment)

train_gen = datagen.flow_from_dataframe(dataframe = train_df,
                                        directory = img_folder,
                                        x_col = 'file_name',
                                        y_col = 'misogynous',
                                        subset = 'training',
                                        batch_size = BATCH_SIZE,
                                        seed = 1,
                                        color_mode = 'rgb',
                                        shuffle = True,
                                        class_mode = 'raw',
                                        target_size = (IMAGE_SIZE, IMAGE_SIZE))

valid_gen = datagen.flow_from_dataframe(dataframe = train_df,
                                        directory = img_folder,
                                        x_col = 'file_name',
                                        y_col = 'misogynous',
                                        subset = 'validation',
                                        batch_size = BATCH_SIZE,
                                        seed = 1,
                                        color_mode = 'rgb',
                                        shuffle = False,
                                        class_mode = 'raw',
                                        target_size = (IMAGE_SIZE, IMAGE_SIZE))

test_gen = datagen.flow_from_dataframe(dataframe = val_df,
                                        directory = img_folder,
                                        x_col = 'file_name',
                                        y_col = 'misogynous',
                                        batch_size = BATCH_SIZE,
                                        seed = 1,
                                        color_mode = 'rgb',
                                        shuffle = False,
                                        class_mode = 'raw',
                                        target_size = (IMAGE_SIZE, IMAGE_SIZE))


# + id="UAr1VIMTRe8X" executionInfo={"status": "ok", "timestamp": 1642108645086, "user_tz": -330, "elapsed": 1, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}}
datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale = 1./255,
                                                          samplewise_center = True,
                                                          samplewise_std_normalization = True)

# + colab={"base_uri": "https://localhost:8080/"} id="iFZVicgfQyfj" executionInfo={"status": "ok", "timestamp": 1642108647834, "user_tz": -330, "elapsed": 379, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="44c99716-2f19-40e9-f0d4-6c8a3e7e1e9e"
test_gen = datagen.flow_from_dataframe(dataframe = test_df,
                                        directory = img_folder,
                                        x_col = 'file_name',
                                        y_col = None,
                                        batch_size = BATCH_SIZE,
                                        seed = 1,
                                        color_mode = 'rgb',
                                        shuffle = False,
                                        class_mode = None,
                                        target_size = (IMAGE_SIZE, IMAGE_SIZE))


# + id="HAYJlF9e7g6t" executionInfo={"status": "ok", "timestamp": 1642108658739, "user_tz": -330, "elapsed": 3561, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}}
# !pip install --quiet vit-keras

from vit_keras import vit

# + id="m5V4sSfsQ7if"
#### seee classes 1 or 2

# + id="x98Jmb7M7g_I" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1640285933061, "user_tz": -330, "elapsed": 17877, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="9100c840-8219-4908-a53c-fee5c8231ca2"
vit_model = vit.vit_b32(
        image_size = IMAGE_SIZE,
        activation = 'sigmoid',
        pretrained = True,
        include_top = False,
        pretrained_top = False,
        classes=1)

# + colab={"base_uri": "https://localhost:8080/"} id="F16yBu6dqNwL" executionInfo={"status": "ok", "timestamp": 1640285935647, "user_tz": -330, "elapsed": 388, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="94ce6eb4-fc13-4758-fcf5-0ded47bedd6f"


# + id="8w_CVjKiG394" colab={"base_uri": "https://localhost:8080/", "height": 356} executionInfo={"status": "error", "timestamp": 1640285959687, "user_tz": -330, "elapsed": 5171, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="a62e807b-fc9f-4a1e-ea79-9011c00a92e2"
from vit_keras import visualize

x = test_gen.next()
image = x[0]

attention_map = visualize.attention_map(model = vit_model, image = image)

# Plot results
fig, (ax1, ax2) = plt.subplots(ncols = 2)
ax1.axis('off')
ax2.axis('off')
ax1.set_title('Original')
ax2.set_title('Attention Map')
_ = ax1.imshow(image)
_ = ax2.imshow(attention_map)

# + id="su9N-irNG4AA" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1640285987220, "user_tz": -330, "elapsed": 2052, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="2acf5066-5d76-4a17-ebb1-7159b0d48a9d"
model = tf.keras.Sequential([
        vit_model,
        tf.keras.layers.Flatten(),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Dense(11, activation = tfa.activations.gelu),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Dense(1, 'sigmoid')
    ],
    name = 'vision_transformer')

model.summary()

# + id="rPUXHcING4Du" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1640317796444, "user_tz": -330, "elapsed": 31787173, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="8464604d-ef55-4542-aa9e-b493b5380ff1"
learning_rate = 1e-4

optimizer = tfa.optimizers.RectifiedAdam(learning_rate = learning_rate)

model.compile(optimizer = optimizer, 
              loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.2), 
              metrics = ['accuracy'])

STEP_SIZE_TRAIN = train_gen.n // train_gen.batch_size
STEP_SIZE_VALID = valid_gen.n // valid_gen.batch_size

reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_accuracy',
                                                 factor = 0.2,
                                                 patience = 2,
                                                 verbose = 1,
                                                 min_delta = 1e-4,
                                                 min_lr = 1e-6,
                                                 mode = 'max')

earlystopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_accuracy',
                                                 min_delta = 1e-4,
                                                 patience = 5,
                                                 mode = 'max',
                                                 restore_best_weights = True,
                                                 verbose = 1)
checkpointer = tf.keras.callbacks.ModelCheckpoint(filepath = './saved_models/vit_binary_image_checkpoint.hdf5',
                                                  monitor = 'val_accuracy', 
                                                  verbose = 1, 
                                                  save_best_only = True,
                                                  save_weights_only = True,
                                                  mode = 'max')

callbacks = [earlystopping, reduce_lr, checkpointer]

model.fit(x = train_gen,
          steps_per_epoch = STEP_SIZE_TRAIN,
          validation_data = valid_gen,
          validation_steps = STEP_SIZE_VALID,
          epochs = EPOCHS,
          callbacks = callbacks)
model.save('./saved_models/vit_binary_image_model.h5')

# + id="6QAy-Wv-RHdG" executionInfo={"status": "ok", "timestamp": 1642108667939, "user_tz": -330, "elapsed": 402, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}}
from keras.models import load_model

# + id="glvPuHqkQCfI" executionInfo={"status": "ok", "timestamp": 1642108687177, "user_tz": -330, "elapsed": 17965, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}}
model=load_model('./saved_models/vit_binary_image_model.h5')

# + id="7O7zbb1vG4Gm" colab={"base_uri": "https://localhost:8080/", "height": 234} executionInfo={"status": "error", "timestamp": 1640348590855, "user_tz": -330, "elapsed": 510609, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="b93dfcb6-58c3-4842-f21b-77b55c1c8423"
predicted_classes = np.round(model.predict(test_gen, steps = test_gen.n // test_gen.batch_size + 1))
true_classes = test_gen.classes
class_labels = list(test_gen.class_indices.keys())  

confusionmatrix = confusion_matrix(true_classes, predicted_classes)
plt.figure(figsize = (16, 16))
sns.heatmap(confusionmatrix, cmap = 'Blues', annot = True, cbar = True)

print(classification_report(true_classes, predicted_classes))

# + id="GJTC_ajNR_77" executionInfo={"status": "ok", "timestamp": 1642108912627, "user_tz": -330, "elapsed": 220666, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}}
outputs = np.round(model.predict(test_gen, steps = test_gen.n // test_gen.batch_size + 1))
outputs=list(map(int,outputs))
predictions_db = pd.DataFrame(data=test_df['file_name'])
predictions_db['misogynist'] = outputs
predictions_db.to_csv('./final_res/answer9.txt', index=False, sep='\t', header=False)

# + id="sDmbQ4ZX7hCA" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1640349419845, "user_tz": -330, "elapsed": 405, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="c1537a34-3326-4d20-abd4-dd87eca2ce48"
predicted_classes

# + colab={"base_uri": "https://localhost:8080/"} id="7jhU0QXBccUx" executionInfo={"status": "ok", "timestamp": 1640349493467, "user_tz": -330, "elapsed": 6, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="a6dd8d08-6ddb-438c-938f-edd9423556a5"
print(classification_report(val_df.misogynous.values, predicted_classes))

# + id="yiUFoG3ecuOh"
from sklearn.metrics import f1_score

# + id="a1RF7nr8c81n"
mi=f1_score(val_df.misogynous.values,predicted_classes,average='macro')

# + colab={"base_uri": "https://localhost:8080/"} id="H04BmJDydMcw" executionInfo={"status": "ok", "timestamp": 1640349631579, "user_tz": -330, "elapsed": 380, "user": {"displayName": "<NAME> 19210068", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Giseq-GUEEEHxpbIRj6F3acRsPjOAw-9WWZZRWdDw=s64", "userId": "10632211144139647575"}} outputId="a6eb113a-b06d-4fa9-c5dc-9d0f1c5ede4a"
mi

# + id="yTJnKHfxdNS_"


StarcoderdataJupyterScriptsDedupFiltered