# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.9.7 64-bit
#     name: python3
# ---

# +
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import itertools
import collections

from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier

from sklearn.metrics import confusion_matrix,f1_score
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
import re
import nltk
from wordcloud import WordCloud

# -

df=pd.read_csv('english_dataset.tsv',sep='\t')

df.head(15)

# ### About the Target Variables:
# **task_1:** Classiﬁcation of Hate Speech (HOF) and non-hate/offensive content.
#
# **task_2:** If the post is HOF, task_2 is used to classify it as either hate speech (HATE), offensive content (OFFN) or profanity (PRFN).
#
# **task_3:** It decides the target of the post, i.e., targeted insult (TIN) or untargeted (UNT).

# # Preprocessing

df=df.drop("text_id", axis=1)

# Removed the column "text_id" as it is of no significance.

df.info()

# #### Checking for Missing Values

df.isna().sum()

# There are no missing values.

from nltk.corpus import stopwords

nltk.download('stopwords')
eng_stops = set(stopwords.words("english"))

nltk.download('wordnet')

from nltk.stem import WordNetLemmatizer 
lemmatizer = WordNetLemmatizer() 


# ### Cleaning Tweets: Removing special characters, converting all letters to lowercase, removing stopwords and lemmatizing.

def process_tweet(tweets):
    # remove all the special characters
    new_tweets = re.sub("[^a-zA-Z]", " ",tweets) 
    # convert all letters to lower case
    words = tweets.lower().split()
    # remove stopwords
    words = [w for w in words if not w in eng_stops]    
    # lemmatizer 
    words = [lemmatizer.lemmatize(word) for word in words]
    # join all words back to text
    return (" ".join(words))


df['clean_tweets']=df['text'].apply(lambda x: process_tweet(x))

df.head(10)

# # Exploratory Data Analysis

df['length'] =  df['clean_tweets'].apply(len)
fig1 = sns.barplot('task_1', 'length', data= df)
plt.title('Average character count vs whether the tweet is Non-hate/Offensive (Not) or Hate and Offensive (HOF)')
plt.show()

# Hence, there is no relation between the character count and type of tweet.

# ### Checking the distribution of samples (i.e, checking for class imbalance)

df.task_1.value_counts().plot(kind="bar", color=[ "lightblue", "purple"]);

# Clearly, we have an **unbalanced target column.**
#
# This needs to be taken into consideration while training the model.

# +
df_hof = df[df["task_1"]=="HOF"]

df_hof.task_2.value_counts().plot(kind="bar", color=[ "red", "purple", "yellow"]);
# -

# Majority of the HOF tweets are **hate** tweets followed by tweets containing profanity and offensive content. 

df_hof.task_3.value_counts().plot(kind="bar", color=["lightblue", "red", "lightgreen"]);

# Most of the hate and offensive tweets are **targeted insults** and only about 12.5% are untargeted.

hof_tweets= df_hof.clean_tweets
hof_words = ' '.join(hof_tweets)

# ### Plotting the wordcloud for the words that appear the most in hate and offensive tweets

wordcloud = WordCloud(width=800, height=500, random_state=21, max_font_size=110).generate(hof_words)
plt.figure(figsize=(10, 7))
plt.imshow(wordcloud, interpolation="bilinear")
plt.title('Frequently used words in Hate/ Offensive tweets', size=20)
plt.axis('off')
plt.show()

df_tin=df[df["task_3"]=="TIN"]
df_tin.task_2.value_counts().plot(kind="bar",color=["red","black","blue"])

# ### This Plot shows the Hate Profanity against Targeted Units.

df_unt=df[df["task_3"]=="UNT"]
df_unt.task_2.value_counts().plot(kind="bar",color=["red","black","blue"])

# ### This Plot shows the Hate Profanity and Offensive tweets against Untargeted units.
#
# It is evident that Profanity to Hate ratio against untargeted units is greater than that of targeted units.

words_in_tweet = [tweet.lower().split() for tweet in df["clean_tweets"]]
words_in_tweet[:2]


# +
# List of all words across tweets
all_words_no_urls = list(itertools.chain(*words_in_tweet))

# Create counter
counts_no_urls = collections.Counter(all_words_no_urls)

counts_no_urls.most_common(15)


# +
clean_tweets_no_urls = pd.DataFrame(counts_no_urls.most_common(15),
                                    columns=['words', 'count'])

clean_tweets_no_urls.head()


# +
fig, ax = plt.subplots(figsize=(8, 8))

# Plot horizontal bar graph
clean_tweets_no_urls.sort_values(by='count').plot.barh(x='words',
                                                       y='count',
                                                       ax=ax,
                                                       color="purple")

ax.set_title("Common Words Found in Tweets (Including All Words)")

plt.show()


StarcoderdataJupyterScriptsDedupFiltered