# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import re
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from wordcloud import WordCloud,STOPWORDS

import nltk
import unidecode
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.stem.porter import *

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import LabelEncoder

# -

data= pd.read_csv("Tweets.csv",encoding = 'ISO-8859-1')
data.head()

print("Percentage null or na values in df")
((data.isnull() | data.isna()).sum() * 100 / data.index.size).round(2)

del data['tweet_coord']
del data['airline_sentiment_gold']
del data['negativereason_gold']


# +
#Removing Twitter Handles
def remove_pattern(input_txt, pattern):
    r = re.findall(pattern, input_txt)
    for i in r:
        input_txt = re.sub(i, '', input_txt)
        
    return input_txt

# remove twitter handles (@...)
data['tidy_text'] = np.vectorize(remove_pattern)(data['text'], "@[\w]*")

# remove special characters, numbers, punctuations
data['tidy_text'] = data['tidy_text'].str.replace("[^a-zA-Z#]", " ")

#Remove greek or unknown characters if any
data['tidy_text'] = data['tidy_text'].apply(lambda x : ' '.join([unidecode.unidecode(word) for word in x.split()]))

#replace with obvious words
d = {'luv':'love','wud':'would','lyk':'like','wateva':'whatever','ttyl':'talk to you later',
               'kul':'cool','fyn':'fine','omg':'oh my god!','fam':'family','bruh':'brother',
               'cud':'could','fud':'food','ve':'have', 'didn':'did not'} ## Need a huge dictionary
data['tidy_text'] = data['tidy_text'].apply(lambda x : ' '.join(d[word] if word in d else word for word in x.split()))

#Removing stop words like a, an, the etc
data['tidy_text'] = data['tidy_text'].apply(lambda x : ' '.join([word for word in x.split() if not word in set(stopwords.words('english'))]))

#Stemming and Lemmatizing
lemmatizer = WordNetLemmatizer()
data['tidy_text'] = data['tidy_text'].apply(lambda x : ' '.join([lemmatizer.lemmatize(word) for word in x.split()]))
ps = PorterStemmer()
data['tidy_text'] = data['tidy_text'].apply(lambda x : ' '.join([ps.stem(word) for word in x.split()]))

#Tokenization
corpus = []
for i in range(0,14640):
    tweet = data['tidy_text'][i]
    tweet = tweet.lower()
    tweet = tweet.split()
    tweet = [word for word in tweet if not word in set(stopwords.words('english'))]
    tweet = ' '.join(tweet)
    corpus.append(tweet)
    
data[['text','tidy_text']].head()

# +

sentiment = []
for i in range(0,len(data['airline_sentiment'])): 
    if data['airline_sentiment'][i]=='negative':
        sentiment.append(0)
    if data['airline_sentiment'][i]=='positive':
        sentiment.append(1)
    if data['airline_sentiment'][i]=='neutral':
        sentiment.append(2)

data['sentiment'] = sentiment
data
# -

sns.countplot(x=data['airline_sentiment'], palette="Set3")
plt.xticks(rotation =90)
plt.xlabel('Sentiments')
plt.ylabel('Tweets Count')
plt.title('Sentiments Count')
plt.show()

# +

# Count of reasons for negative tweets for airlines, overall customer service issue was major problem
sns.countplot(x=data['negativereason'], palette="Set3")
plt.xticks(rotation =90)
plt.show()

# +

# Count of reasons for negative tweets for airlines, overall customer service issue was major problem
sns.countplot(x=data['negativereason'], palette="Set3")
plt.xticks(rotation =90)
plt.show()

# +
# Converting datetime column to pandas datetime format and keeping only date
data['date'] = pd.to_datetime(data['tweet_created'])
data['date'] = data['date'].apply(lambda x : x.date())

df1 = data.groupby(['date','airline','airline_sentiment']).size()
df1 = df1.loc(axis=0)[:,:,'negative']
axis = df1.groupby(['date', 'airline']).sum().unstack().plot(kind='bar', figsize=(12,8), rot=45)
axis.set_ylabel('Negative Tweets Count')
axis.legend(loc='upper left', bbox_to_anchor = (1,1))

# +
new_df=data[data['airline_sentiment']=='negative']
words = ' '.join(new_df['tidy_text'])

wordcloud = WordCloud(stopwords=STOPWORDS,
                      background_color='white',
                      width=3000,
                      height=2500
                     ).generate(words)
plt.figure(1,figsize=(12, 12))
plt.imshow(wordcloud)
plt.axis('off')
plt.show()
# -

from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer  
cv = CountVectorizer(max_df=0.90, min_df=2, stop_words='english')  
X = cv.fit_transform(corpus).toarray()
X

sentiment = []
for i in range(0,len(data['airline_sentiment'])): 
    if data['airline_sentiment'][i]=='negative':
        sentiment.append(0)
    if data['airline_sentiment'][i]=='positive':
        sentiment.append(2)
    if data['airline_sentiment'][i]=='neutral':
        sentiment.append(1)
data['sentiment'] = sentiment
Y = data['sentiment'].values
Y

from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.20, random_state = 0)

# # KNN

# try K=1 through K=15 and record testing accuracy
k_range = list(range(1, 16))
scores = []
for k in k_range:
    knn = KNeighborsClassifier(n_neighbors=k)
    knn.fit(X_train, Y_train)
    y_pred = knn.predict(X_test)
    scores.append(metrics.accuracy_score(Y_test, y_pred))

plt.plot(k_range, scores)
plt.xlabel('Value of K')
plt.ylabel('Testing Accuracy')

#With k = 2
kn = KNeighborsClassifier(n_neighbors=2)
model_knn = kn.fit(X_train,Y_train)
k_pred = model_knn.predict(X_test)
print(metrics.accuracy_score(Y_test,k_pred))

from sklearn.metrics import confusion_matrix
cm = confusion_matrix(Y_test,k_pred)
print(cm)

# # Logistic Regression

# +
from sklearn.linear_model import LogisticRegression
X3_train, X3_test, Y3_train, Y3_test = train_test_split(data['tidy_text'],data['sentiment'], test_size = 0.2, random_state=0, shuffle = True, stratify=data['sentiment'])

from sklearn.pipeline import Pipeline
from sklearn.metrics import confusion_matrix,classification_report
#Training and testing the randomforest model
rf = Pipeline([('tfidf', TfidfVectorizer()), ('rf', LogisticRegression())])
rf.fit(X3_train, Y3_train)
y_pred = rf.predict(X3_test)
print(pd.crosstab(Y3_test,y_pred,rownames=['Actual'],colnames=['Predicted']))
print(classification_report(Y3_test, y_pred))

# +
#Logistic Regression using 10 fold - cross validation
lr = LogisticRegression()
scores = cross_val_score(lr, X, Y, cv=10, scoring='accuracy')
print(scores)
s0 = scores.mean()

#Logistic Regression using 5 fold - cross validation

lr = LogisticRegression()
scores = cross_val_score(lr, X, Y, cv=5, scoring='accuracy')
print(scores)
s1 = scores.mean()

#Logistic Regression using 15 fold - cross validation

lr = LogisticRegression()
scores = cross_val_score(lr, X, Y, cv=15, scoring='accuracy')
print(scores)
s2 = scores.mean()

#Logistic Regression using 20 fold - cross validation

lr = LogisticRegression()
scores = cross_val_score(lr, X, Y, cv=20, scoring='accuracy')
print(scores)
s3 = scores.mean()

#Taking the average of 4 cross validated accuracies
print('Average accuracy of Logistic Regression is '+ str((s0 + s1 + s2 + s3)/4))
# -

# # Random Forest

# +
from sklearn.ensemble import RandomForestClassifier

#1 - RandomForest classification with Pipeline
X2_train, X2_test, Y2_train, Y2_test = train_test_split(data['tidy_text'],data['sentiment'], test_size = 0.2, random_state=0, shuffle = True, stratify=data['sentiment'])


#Training and testing the randomforest model
rf = Pipeline([('tfidf', TfidfVectorizer()), ('rf', RandomForestClassifier())])
rf.fit(X2_train, Y2_train)
y_pred = rf.predict(X2_test)
print(pd.crosstab(Y2_test,y_pred,rownames=['Actual'],colnames=['Predicted']))
print(classification_report(Y2_test, y_pred))

# +
#Random Forest using 5 fold Cross validation with 
rf = RandomForestClassifier()
scores = cross_val_score(rf, X, Y, cv=5, scoring='accuracy')
print(scores)
s0 = scores.mean()

#Random Forest using 10 fold Cross validation with 
rf = RandomForestClassifier()
scores = cross_val_score(rf, X, Y, cv=10, scoring='accuracy')
print(scores)
s1 = scores.mean()

#Random Forest using 15 fold Cross validation with 
rf = RandomForestClassifier()
scores = cross_val_score(rf, X, Y, cv=15, scoring='accuracy')
print(scores)
s2 = scores.mean()

#Random Forest using 15 fold Cross validation with 
rf = RandomForestClassifier()
scores = cross_val_score(rf, X, Y, cv=20, scoring='accuracy')
print(scores)
s3 = scores.mean()

print('Average accuracy of Random Forest is '+ str((s0 + s1 + s2 + s3)/4))
# -



StarcoderdataJupyterScriptsDedupFiltered