<jupyter_start><jupyter_text>Initial data exploration---[Memotion dataset 7k](https://www.kaggle.com/williamscott701/memotion-dataset-7k): Dataset for sentiment classification of memes<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
import pickle
from PIL import Image<jupyter_output><empty_output><jupyter_text>Load<jupyter_code>print('Current directory:', os.getcwd())
print('Changing directory: moving up')
os.chdir('../')
print('Current directory:', os.getcwd())
# Pickling
data_raw_path = "data/raw/"
with open(data_raw_path+"labels_pd_pickle","rb") as file_handle:
    labels_df = pickle.load(file_handle)
    
with open(data_raw_path+"reference_df_pickle","rb") as file_handle:
    reference_df = pickle.load(file_handle)
display(labels_df.head(10))
labels_df.shape
display(reference_df.head(3))
reference_df.shape<jupyter_output><empty_output><jupyter_text>Images<jupyter_code>def plot_image(image_name, figure_path=None):
    image = Image.open(data_raw_path+"images/"+image_name)
    # summarize some details about the image
    print(image.format, image.size, image.mode)
    # show the image
    plt.imshow(image)
    plt.axis('off')
    if figure_path:
        plt.savefig(figure_path, bbox_inches='tight')
    plt.show()
plot_image(labels_df.image_name[0])
labels_df.text_corrected[0]<jupyter_output>JPEG (735, 500) RGB<jupyter_text>Text and labels  Missing values<jupyter_code>labels_df.isna().sum()
labels_na = labels_df[labels_df.isna().any(axis=1)]
pd.set_option('display.max_rows', labels_na.shape[0]+1)
labels_na
labels_na = labels_df[labels_df.text_corrected.isna()]
pd.set_option('display.max_rows', labels_na.shape[0]+1)
labels_na
# Explore the 5 images with NaN text_corrected 
for i, row in labels_na.iterrows():
    print(i)
    plot_image(row.image_name)<jupyter_output>119
JPEG (500, 480) RGB<jupyter_text>Drop NA<jupyter_code>labels_df = labels_df.dropna()<jupyter_output><empty_output><jupyter_text>Labels distribution<jupyter_code>labels_df.describe()
colors = ['C0', 'C1', 'C2', 'C3', 'C4']
_ = labels_df.humour.value_counts().plot(kind='bar', figsize=(10,5), rot=0, color=colors)
plt.savefig('reports/figures/humour_distribution.png', bbox_inches='tight')
_ = labels_df.sarcasm.value_counts().plot(kind='bar', figsize=(10,5), rot=0, color=colors)
plt.savefig('reports/figures/sarcasm_distribution.png', bbox_inches='tight')
_ = labels_df.offensive.value_counts().plot(kind='bar', figsize=(10,5), rot=0, color=colors)
plt.savefig('reports/figures/offensive_distribution.png', bbox_inches='tight')
_ = labels_df.motivational.value_counts().plot(kind='bar', figsize=(10,5), rot=0, color=colors)
plt.savefig('reports/figures/motivational_distribution.png', bbox_inches='tight')
_ = labels_df.overall_sentiment.value_counts().plot(kind='bar', figsize=(10,5), rot=0, color=colors)
plt.savefig('reports/figures/overall_sentiment_distribution.png', bbox_inches='tight')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup