# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# cd "C:\Users\Naman\Downloads\AnalytixLab\Final Project\5. bank reviews-complaints analysis"

pwd

import pandas as pd
import numpy as np

dataset=pd.read_csv('Bank_Reviews_csv.csv')

dataset.head()

dataset.info()

dataset.shape

dataset.drop(['Unnamed: 0','Date','BankName'],axis=1,inplace=True)

dataset.head()

dataset.info()

dataset.shape

dataset.columns=['STARS','REVIEWS']

text_data=dataset.REVIEWS.values.tolist()

text_data

len(text_data)

import re

# Creating the corpus along with structuring data in a readable form
corpus = []
for review in text_data:
    review = review.lower()
    review = re.sub(r"that's","that is",review)
    review = re.sub(r"there's","there is",review)
    review = re.sub(r"what's","what is",review)
    review = re.sub(r"where's","where is",review)
    review = re.sub(r"it's","it is",review)
    review = re.sub(r"who's","who is",review)
    review = re.sub(r"i'm","i am",review)
    review = re.sub(r"she's","she is",review)
    review = re.sub(r"he's","he is",review)
    review = re.sub(r"they're","they are",review)
    review = re.sub(r"who're","who are",review)
    review = re.sub(r"ain't","am not",review)
    review = re.sub(r"wouldn't","would not",review)
    review = re.sub(r"shouldn't","should not",review)
    review = re.sub(r"can't","can not",review)
    review = re.sub(r"couldn't","could not",review)
    review = re.sub(r"won't","will not",review)
    review = re.sub(r"\W"," ",review)
    review = re.sub(r"\d"," ",review)
    review = re.sub(r"\s+[a-z]\s+"," ",review)
    review = re.sub(r"\s+[a-z]$"," ",review)
    review = re.sub(r"^[a-z]\s+"," ",review)
    review = re.sub(r"\s+"," ",review)
    review = re.sub(r'^br$', ' ', review)
    review = re.sub(r'\W', ' ',review)
    review = review.rstrip() #to remove trailing whitespaces at the end of the string
    review = review.lstrip() #to remove leading whitespaces at the start of the string
    corpus.append(review) 

corpus

len(corpus)

# +
#string of the corpus 
str_representation = ' '.join(corpus)

#list of words of the corpus
list_words=str_representation.split(" ")
print('Number of words in the corpus is : ',len(list_words))

#unique count of words
print('Number of unique words in the corpus are : ',len(set(list_words)))
# -

import nltk

# +
## Removing stopwords( here i am doing it before lemmetization as latter converts words like 'has' to 'ha' and so if stop words 
## removal done after lemmetization then 'ha' wont be captured)

from nltk.corpus import stopwords

for i in range(len(corpus)):
    words = nltk.word_tokenize(corpus[i])
    words = [word for word in words if word not in stopwords.words('english')]
    corpus[i] = ' '.join(words)
# -

len(corpus)

corpus[:7]

# +
#string of the corpus after removing stopwords
str_representation_stop = ' '.join(corpus)

#list of words of the corpus after removing stopwords
list_words_stop=str_representation_stop.split(" ")
print('Number of words in the corpus is : ',len(list_words_stop))

#unique count of words after removing stopwords
print('Number of unique words in the corpus are : ',len(set(list_words_stop)))

# +
#lemmetization (note : numner of words after lemmetization will remain same as number of words after stop words.Only difference
#will be the number of unique words)

from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()

for i in range(len(corpus)):
    words = nltk.word_tokenize(corpus[i])
    words = [lemmatizer.lemmatize(word) for word in words]
    corpus[i] = ' '.join(words)   
# -

len(corpus)

corpus[:7]

# +
#string of the corpus after lemmetization
str_representation_lem = ' '.join(corpus)

#list of words of the corpus after lemmetization
list_words_lem=str_representation_lem.split(" ")
print('Number of words in the corpus is : ',len(list_words_lem))

#unique count of words after lemmetization
print('Number of unique words in the corpus are : ',len(set(list_words_lem)))

# +
# Again Removing stopwords ( Here I am doing after lemmetization as if after lemmetizing the words if any word comes in the
# category of stopwords, it will be removed)
from nltk.corpus import stopwords

for i in range(len(corpus)):
    words = nltk.word_tokenize(corpus[i])
    words = [word for word in words if word not in stopwords.words('english')]
    corpus[i] = ' '.join(words)
# -

len(corpus)

# +
#string of the corpus after removing stopwords
str_representation_stop_again = ' '.join(corpus)

#list of words of the corpus after removing stopwords
list_words_stop_again=str_representation_stop_again.split(" ")
print('Number of words in the corpus is : ',len(list_words_stop_again))

#unique count of words after removing stopwords
print('Number of unique words in the corpus are : ',len(set(list_words_stop_again)))
# -

corpus

# Creating word histogram to get count of each word
word2count = {}
for data in corpus:
    words = nltk.word_tokenize(data)
    for word in words:
        if word not in word2count.keys():
            word2count[word] = 1
        else:
            word2count[word] += 1

word2count

# Selecting most occuring words
import heapq
freq_words = heapq.nlargest(400,word2count,key=word2count.get)

len(freq_words)

freq_words_corpus=corpus[:]

len(freq_words_corpus)

# +
#string of the freq_words_corpus
str_freq_words_corpus = ' '.join(freq_words_corpus)

#list of words of the corpus after removing stopwords
list_freq_words_corpus=str_freq_words_corpus.split(" ")
print('Number of words in the corpus is : ',len(list_freq_words_corpus))

#unique count of words after removing stopwords
print('Number of unique words in the corpus are : ',len(set(list_freq_words_corpus)))
# -

#creating corpus of frequent words
for i in range(len(freq_words_corpus)):
    words = nltk.word_tokenize(freq_words_corpus[i])
    words = [word for word in words if word in freq_words]
    freq_words_corpus[i] = ' '.join(words)

len(freq_words_corpus)

# +
#string of the corpus after only keeping freq words
str_representation_freq_words= ' '.join(freq_words_corpus)

#list of words of the corpus after only keeping freq words
list_words_freq_words=str_representation_freq_words.split(" ")
print('Number of words in the corpus is : ',len(list_words_freq_words))

#unique count of words after only keeping freq words
print('Number of unique words in the corpus are : ',len(set(list_words_freq_words)))
# -

from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(stop_words='english')
X = vectorizer.fit_transform(freq_words_corpus)

len(vectorizer.get_feature_names())

Review=pd.DataFrame(X.toarray())

Review.head()

terms = vectorizer.get_feature_names()

Review.columns=terms

Review.head()

Review.info()

dataset.head()

updated_dataset=pd.DataFrame()

updated_dataset=pd.concat([updated_dataset,Review],axis=1)

updated_dataset.head()

updated_dataset.head()

updated_dataset.info()

updated_dataset.shape

updated_dataset=pd.concat([updated_dataset,dataset['STARS']],axis=1)

updated_dataset.STARS=np.where(updated_dataset.STARS==5,1,0)

updated_dataset.head()

updated_dataset.info()

updated_dataset.shape

from sklearn.cross_validation import train_test_split

rf_feature_columns=updated_dataset.columns.difference(['STARS'])

len(rf_feature_columns)

rf_train_X, rf_test_X, rf_train_y, rf_test_y = train_test_split( updated_dataset[rf_feature_columns],
                                                  updated_dataset['STARS'],
                                                  test_size = 0.2,
                                                  random_state = 42 )

rf_train_X.head()

rf_train_X.shape

rf_train_y.head()

rf_train_y.shape

rf_test_X.head()

rf_test_X.shape

rf_test_y.head()

rf_test_y.shape

from sklearn.ensemble import RandomForestClassifier

rf_tree = RandomForestClassifier(oob_score=True,n_estimators=116,max_features=105)
rf_tree.fit( rf_train_X, rf_train_y )

radm_test_pred = pd.DataFrame( { 'actual':  rf_test_y,
                            'predicted': rf_tree.predict( rf_test_X ) } )

radm_test_pred.to_excel('Actual_vs_Predicted.xlsx')

from sklearn import metrics

metrics.accuracy_score( radm_test_pred.actual, radm_test_pred.predicted )

##Parameter tuning for random forest
param_grid={'n_estimators':np.arange(100,150),
           'max_features':np.arange(100,200)}

from sklearn.tree import DecisionTreeClassifier, export_graphviz, export
from sklearn.grid_search import GridSearchCV

rf_tree = GridSearchCV(RandomForestClassifier(oob_score=False,warm_start=True,n_jobs=-1), param_grid, cv = 10)
rf_tree.fit( rf_train_X, rf_train_y )

rf_tree.best_estimator_

rf_tree.best_score_

# ### CONFUSION MATRIX

from sklearn.metrics import  confusion_matrix

cm=confusion_matrix(radm_test_pred.actual,radm_test_pred.predicted)

cm

import seaborn as sn
import matplotlib.pyplot as plt

sn.heatmap(cm, annot=True, fmt='.0f',xticklabels=["1",'5'],yticklabels=["1",'5'])
plt.ylabel('ACTUAL')
plt.xlabel('PREDICTED')
plt.title('Test Data Confusion Matrix')
plt.show()


# ### Finding Ideal Cutoff

test_predicted_prob = pd.DataFrame(rf_tree.predict_proba(rf_test_X))[[1]]
test_predicted_prob.columns = ['prob']
actual=rf_test_y.reset_index()
actual.drop('index',axis=1,inplace=True)
# making a DataFrame with actual and prob columns
hr_test_predict = pd.concat([actual, test_predicted_prob], axis=1)
hr_test_predict.columns = ['actual','prob']
hr_test_predict.head()

hr_test_predict.head()

hr_test_predict.info()

# +
test_roc_like_df = pd.DataFrame()
test_temp = hr_test_predict.copy()

for cut_off in np.linspace(0,1,50):
    test_temp['predicted'] = test_temp['prob'].apply(lambda x: 0 if x < cut_off else 1)
    test_temp['tp'] = test_temp.apply(lambda x: 1 if x['actual']==1 and x['predicted']==1 else 0, axis=1)
    test_temp['fp'] = test_temp.apply(lambda x: 1 if x['actual']==0 and x['predicted']==1 else 0, axis=1)
    test_temp['tn'] = test_temp.apply(lambda x: 1 if x['actual']==0 and x['predicted']==0 else 0, axis=1)
    test_temp['fn'] = test_temp.apply(lambda x: 1 if x['actual']==1 and x['predicted']==0 else 0, axis=1)
    sensitivity = test_temp['tp'].sum() / (test_temp['tp'].sum() + test_temp['fn'].sum())
    specificity = test_temp['tn'].sum() / (test_temp['tn'].sum() + test_temp['fp'].sum())
    test_roc_like_table = pd.DataFrame([cut_off, sensitivity, specificity]).T
    test_roc_like_table.columns = ['cutoff', 'sensitivity', 'specificity']
    test_roc_like_df = pd.concat([test_roc_like_df, test_roc_like_table], axis=0)
# -

test_roc_like_df.head()

test_roc_like_df.info()

test_temp.sum()
plt.subplots(figsize=(10,4))
plt.scatter(test_roc_like_df['cutoff'], test_roc_like_df['sensitivity'], marker='*', label='Sensitivity')
plt.scatter(test_roc_like_df['cutoff'], test_roc_like_df['specificity'], marker='*', label='Specificity')
#plt.scatter(test_roc_like_df['cutoff'], 1-test_roc_like_df['specificity'], marker='*', label='FPR')
plt.title('For each cutoff, pair of sensitivity and FPR is plotted for ROC')
plt.legend()

## Finding ideal cut-off for checking if this remains same in OOS validation
test_roc_like_df['total'] = test_roc_like_df['sensitivity'] + test_roc_like_df['specificity']
test_roc_like_df[test_roc_like_df['total']==test_roc_like_df['total'].max()]

hr_test_predict['predicted'] = hr_test_predict['prob'].apply(lambda x: 1 if x > 0.67 else 0)
hr_test_predict.head()

hr_test_predict.to_excel('Actual_vs_Predicted.xlsx')

# +
### CONFUSION MATRIX WITH IDEAL CUT-OFF
# -

cm=confusion_matrix(hr_test_predict.actual,hr_test_predict.predicted)

sn.heatmap(cm, annot=True, fmt='.0f',xticklabels=["1",'5'],yticklabels=["1",'5'])
plt.ylabel('ACTUAL')
plt.xlabel('PREDICTED')
plt.title('Test Data Confusion Matrix with Ideal Cut-Off')
plt.show()


## Accuracy
((20+73)/(20+73+7+1))

##Precision
20/27

##Recall
20/21



StarcoderdataJupyterScriptsDedupFiltered