<jupyter_start><jupyter_text>E Commerce Review Analysis  Importing Libraries<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
import warnings
import time<jupyter_output><empty_output><jupyter_text>Working on dataset<jupyter_code>#Importing data from CVS
data = pd.read_csv('Womens_Clothing_E_Commerce_Reviews.csv')
#Leaving of first column as we have index
data.drop('Unnamed: 0', axis=1,inplace=True)
#Removing rows with nan
data.dropna(subset=['Review Text','Division Name','Department Name','Class Name'],inplace=True)
data.columns = ['Clothing_Id', 'Age', 'Title', 'Review', 'Rating', 'Recommended_IND', 'Positive_Feedback_Count','Division_Name', 'Department_Name', 'Class_Name']
data.head(3)
# FInding number of unique things in each column
print('Clothing_Id :' + str(len(list(set(data['Clothing_Id'])))))
print('Age :' + str(len(list(set(data['Age'])))))
print('Rating :' + str(len(list(set(data['Rating'])))))
print('Recommended_IND :' + str(len(list(set(data['Recommended_IND'])))))
print('Positive_Feedback_Count :' + str(len(list(set(data['Positive_Feedback_Count'])))))
print('Division_Name :' + str(len(list(set(data['Division_Name'])))) + str(set(data['Division_Name'])))
print('Department_Name :' + str(len(list(set(data['Department_Name'])))) +  str(set(data['Department_Name'])))
print('Class_Name :' + str(len(list(set(data['Class_Name'])))))<jupyter_output>Clothing_Id :1172
Age :77
Rating :5
Recommended_IND :2
Positive_Feedback_Count :82
Division_Name :3{'General', 'General Petite', 'Initmates'}
Department_Name :6{'Trend', 'Intimate', 'Bottoms', 'Tops', 'Jackets', 'Dresses'}
Class_Name :20<jupyter_text>Analysis 1: Department & Division  v.s.  Review Count<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')

sns.set_style('darkgrid')
f, axes = plt.subplots (1,2, figsize=(18,6))

# Histrogram
x = ['Jackets', 'Bottoms', 'Dresses', 'Trend', 'Tops', 'Intimate']
y = [data.Department_Name[(data['Department_Name']=='Jackets')].count(),
     data.Department_Name[(data['Department_Name']=='Bottoms')].count(),
     data.Department_Name[(data['Department_Name']=='Dresses')].count(),
     data.Department_Name[(data['Department_Name']=='Trend')].count(),
     data.Department_Name[(data['Department_Name']=='Tops')].count(),
     data.Department_Name[(data['Department_Name']=='Intimate')].count(), ]

vis1= sns.barplot(x,y,palette='rocket', ax=axes[0])
vis1.set(xlabel='Department Name',ylabel='Number of Reviews for all Divisiion Names')

for p in vis1.patches:
             vis1.annotate("%.f" % p.get_height(), (p.get_x() + p.get_width() / 2., p.get_height()),
                 ha='center', va='center', fontsize=11, color='black', xytext=(0, 20),
                 textcoords='offset points')


# Histrogram
NG = [data.Division_Name[(data['Division_Name']=='General')].count(), data.Division_Name[(data['Division_Name']=='Initmates')].count(), data.Division_Name[(data['Division_Name']=='General Petite')].count() ]
G = ['General', 'Initmates', 'General Petite']

plt.title('Division Name')
plt.pie(NG, labels=G, startangle=90, autopct='%.1f%%')
plt.ioff()<jupyter_output><empty_output><jupyter_text>Analysis 2: Department & Division  v.s.  Positive Feedback Count<jupyter_code>f, axes = plt.subplots (1,2, figsize=(18,6))

#stripplot Number of positive feedbacks given by people for different Division name giving different ratings
vis1= sns.stripplot(y='Positive_Feedback_Count' , x='Division_Name', data=data,\
                hue='Rating',dodge=True, size=3, ax=axes[0])
vis1.set(ylim=(0, 150))
vis1.set(xlabel='Division Name',ylabel='Positive Feedback Count')

#stripplot Number of positive feedbacks given by people for different Department Name giving different ratings
vis2= sns.stripplot(y='Positive_Feedback_Count' , x='Department_Name', data=data,\
                hue='Rating',dodge=True, size=3, ax=axes[1])
plt.legend(title='User Rating(1-5)',bbox_to_anchor=(1.29, 1))
vis2.set(ylim=(0, 150))
vis2.set(xlabel='Department Name',ylabel='Positive Feedback Count')

plt.ioff()
data.head(4)<jupyter_output><empty_output><jupyter_text>Analysis 3: Positive Feedback Count  v.s.  Age<jupyter_code>#Number of ratings given by people of different age group
plt.subplots(figsize=(20,10))
vis2 = sns.lineplot(x=data.Age, y=data.Positive_Feedback_Count , hue=data.Rating , err_style=None, marker='o')
vis2.set(ylim=(0, 20))
vis2.set_title('Number of ratings given by people of different age group', fontsize=15)

plt.legend(bbox_to_anchor=(1, 1))
plt.ioff()<jupyter_output><empty_output><jupyter_text>Analysis 4: Positive Feedback Count  v.s.  Recommendation<jupyter_code># Rating given by people who recommend and do not recommend products to others
plt.subplots (figsize=(6,6))
vis1= sns.stripplot(y='Positive_Feedback_Count' , x='Recommended_IND', data=data,\
                hue='Rating',dodge=True, size=3)
vis1.set(ylim=(0, 150))
plt.legend(title='User Rating(1-5)',bbox_to_anchor=(1.29, 1))
vis1.set(xlabel='Recommendation',ylabel='Positive Feedback Count')
vis1.set_title('Rating given by people who recommend and \n do not recommend products to others')
plt.show()<jupyter_output><empty_output><jupyter_text>From the above graph we come to know that the people who do not recommend the product to others tend to give low ratingsCompared to those who recommend the products to others  Analysis 5: Recommendation Count  v.s. Age<jupyter_code># Finding out which age group people give the most recomendation to others and also who don't recommend
plt.subplots(figsize=(20,8))
sns.set(style="darkgrid")
ax = sns.countplot(x="Age",hue='Recommended_IND', data=data)<jupyter_output><empty_output><jupyter_text>From the above graph we can come to a conclusion that the age group between 33 to 53 are most active and recommend the most  Creating Rating Class<jupyter_code>data_class = data
data_class['Class'] = 'Bad'
data_class.loc[data_class.Rating>3,['Class']]  = 'Good'
data_class.drop(['Title', 'Age','Rating'], axis=1, inplace=True)
data_class.head(3)
x = data_class['Class'].unique()
y = [data_class.Class[(data_class['Class'] == 'Good')].count(),
     data_class.Class[(data_class['Class'] == 'Bad')].count()]

plt.subplots(figsize=(4,4))
graph = sns.barplot(x,y,palette='rocket')
# vis1= sns.barplot(x,y,palette='rocket')
graph.set(xlabel='Class Name',ylabel='Number of Class for all reviews')
plt.title('Number of Good And Bad Reviews')
plt.ioff()<jupyter_output><empty_output><jupyter_text>Analysis 6: Top 10 most reviewed products<jupyter_code>cloth_id = ((pd.DataFrame({'Number_of_reviews' : (data_class.groupby('Clothing_Id').count()['Class'])})).reset_index(level=0)).sort_values(by=['Number_of_reviews'], ascending=False)[:10]
# """ .plot(x='Clothing_Id', y ='Number_of_reviews',kind='bar',title='Top 10 most reviewed products') """
graph = sns.barplot(x='Clothing_Id', y ='Number_of_reviews',data=cloth_id,order=list(cloth_id['Clothing_Id']),palette='Blues_r')
graph.set_title('Top 10 most reviewed products')
plt.ioff()<jupyter_output><empty_output><jupyter_text>Analysis 7: Good & Bad Reviews in top 10 products<jupyter_code># Plot to show how many people like and dilike a product of top 10 liked products
good_df = (((pd.DataFrame({'Number_of_good_reviews' : ((data_class.loc[data_class.Class == 'Good']).groupby('Clothing_Id').count()['Class'])})).reset_index(level=0)).sort_values(by=['Number_of_good_reviews'], ascending=False))
bad_df = (((pd.DataFrame({'Number_of_bad_reviews' : ((data_class.loc[data_class.Class == 'Bad']).groupby('Clothing_Id').count()['Class'])})).reset_index(level=0)).sort_values(by=['Number_of_bad_reviews'], ascending=False))
review_cloth_id = pd.merge(good_df, bad_df,on=['Clothing_Id'],how='outer').sort_values(by=['Number_of_good_reviews'],ascending=False)[:10]
review_cloth_id.plot(x ='Clothing_Id', kind='bar', stacked=True,title='Good and Bad reviews in top 10 Good products')

plt.ioff()<jupyter_output><empty_output><jupyter_text>Applying Natural Language Processing  Cleaning data for Model<jupyter_code>model_data = data_class
model_data.drop(['Clothing_Id','Recommended_IND','Positive_Feedback_Count','Division_Name','Department_Name','Class_Name'], axis=1, inplace=True)
model_data['Processed_review'] = np.nan
model_data = model_data.reset_index()
model_data.drop(['index'], axis=1,inplace=True)
import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.stem import PorterStemmer, SnowballStemmer
import string

ps = PorterStemmer()
sno = SnowballStemmer('english') 
lemmatizer = WordNetLemmatizer()
stop_words = set(stopwords.words('english'))
# CLeaning and add it to the DataFrame
def process(i):
    orginal_sent = model_data['Review'][i]
    tokenized_sent = word_tokenize(orginal_sent)
    stop_sent = [w for w in tokenized_sent if w not in stop_words and w not in string.punctuation]
    lemmatized_sent = [lemmatizer.lemmatize(w) for w in stop_sent]
    model_data['Processed_review'][i] = ' '.join(lemmatized_sent)

for i in range(len(model_data['Review'])):
    try:
        process(i)
    except:
        pass<jupyter_output><empty_output><jupyter_text>Creating Bag of Words Model<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import naive_bayes
from sklearn.metrics import accuracy_score
from sklearn.ensemble import RandomForestClassifier
from sklearn import tree
from sklearn import svm
import sklearn
# Vectorizing and defining x and y
vectorize = TfidfVectorizer(use_idf = True, lowercase=True, strip_accents='ascii', stop_words=stop_words,max_features=4000)
y = model_data.Class
x = vectorize.fit_transform(model_data.Processed_review)<jupyter_output><empty_output><jupyter_text>Naive Bayes (BernoulliNB)<jupyter_code>def model_BernoulliNB():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y, random_state=421,test_size=0.2)
    # Training the model
    clf = naive_bayes.BernoulliNB()  
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return accuracy_score(y_test,y_pred) * 100, time.time() - start_time<jupyter_output><empty_output><jupyter_text>Naive Bayes (MultinomialNB)<jupyter_code>def model_MultinomialNB():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y, random_state=421)
    # Training the model
    clf = naive_bayes.MultinomialNB()   
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return (accuracy_score(y_test,y_pred) * 100), (time.time() - start_time)<jupyter_output><empty_output><jupyter_text>Naive Bayes (ComplementNB)<jupyter_code>def model_ComplementNB():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y, random_state=421)
    # Training the model
    clf = naive_bayes.ComplementNB() 
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return (accuracy_score(y_test,y_pred) * 100), (time.time() - start_time)<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>def model_RandomForestClassifier():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y, random_state=421)
    # Training the model
    clf = RandomForestClassifier()   
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return (accuracy_score(y_test,y_pred) * 100), (time.time() - start_time)<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>def DecisionTreeClassifier():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y, random_state=421)
    # Training the model
    clf = tree.DecisionTreeClassifier()   
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return (accuracy_score(y_test,y_pred) * 100), (time.time() - start_time)<jupyter_output><empty_output><jupyter_text>SVM (linear kernel)<jupyter_code>def model_svm_linear():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y, random_state=421)
    # Training the model
    clf = svm.SVC(kernel='linear')   
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return (accuracy_score(y_test,y_pred) * 100), (time.time() - start_time)<jupyter_output><empty_output><jupyter_text>SVM (poly kernel)<jupyter_code>def model_svm_poly():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y, random_state=421)
    # Training the model
    clf = sklearn.svm.SVC(kernel='poly')   
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return (accuracy_score(y_test,y_pred) * 100), (time.time() - start_time)<jupyter_output><empty_output><jupyter_text>SVM (rbf kernel)<jupyter_code>def model_svm_rbf():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y, random_state=421)
    # Training the model
    clf = sklearn.svm.SVC(kernel='rbf')   
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return (accuracy_score(y_test,y_pred) * 100), (time.time() - start_time)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>def model_LogisticRegression():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y)
    # Training the model
    clf = sklearn.linear_model.LogisticRegression()   
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return (accuracy_score(y_test,y_pred) * 100), (time.time() - start_time)<jupyter_output><empty_output><jupyter_text>K Nearest Neighbors<jupyter_code>def model_KNeighborsClassifier():
    start_time = time.time()
    # train test split
    x_train, x_test, y_train, y_test = train_test_split(x,y)
    # Training the model
    clf = sklearn.neighbors.KNeighborsClassifier()   
    clf.fit(x_train, y_train)
    # Testing
    y_pred = clf.predict(x_test)
    return (accuracy_score(y_test,y_pred) * 100), (time.time() - start_time)<jupyter_output><empty_output><jupyter_text>Summary of ObservationsNote : The cell below takes a lot of time to execute, as all the models are actually applied here<jupyter_code>models = [model_BernoulliNB(), model_MultinomialNB(), model_ComplementNB(), model_RandomForestClassifier(), DecisionTreeClassifier(), model_svm_linear(), model_svm_poly(), model_svm_rbf(), model_LogisticRegression(), model_KNeighborsClassifier()]

models_name = ['BernoulliNB', 'MultinomialNB', 'ComplementNB', 'RandomForestClassifier', 'DecisionTreeClassifier', 'svm_linear', 'svm_poly', 'svm_rbf', 'LogisticRegression', 'KNeighborsClassifier']

accuracy = []
timing = []
for m in models:
    acc, tim = m
    accuracy.append(acc)
    timing.append(tim)
result = pd.DataFrame({'Models' : models_name, 'Accuracy' : accuracy, 'Timing' : timing})
result
# plotting Accuracy and Timings of all models
f, axes = plt.subplots (1,2, figsize=(18,6))

# Accuracy
result.sort_values(by=['Accuracy'], ascending=False, inplace=True)
result = result.reset_index()
result.drop(['index'], axis=1,inplace=True)

vis1= sns.barplot(y='Accuracy' , x='Models', data=result,dodge=True, ax=axes[0],palette='PuRd_r',)
vis1.set(ylim=(75, 90))
vis1.set(xlabel='Models',ylabel='Accuracy')
vis1.set_xticklabels(vis1.get_xticklabels(), rotation=45)
vis1.set_title('Accuracy of different Models')

for index, row in result.iterrows():
    vis1.text(row.name,row.Accuracy, round(row.Accuracy,2), color='black', ha="center")


# Timing
result.sort_values(by=['Timing'], ascending=False, inplace=True)
result = result.reset_index()
result.drop(['index'], axis=1,inplace=True)

vis2= sns.barplot(y='Timing' , x='Models', data=result,dodge=True, ax=axes[1],palette='Blues_r',)
vis2.set(ylim=(0, 500))
vis2.set(xlabel='Models',ylabel='Runtime in seconds')
vis2.set_xticklabels(vis2.get_xticklabels(), rotation=45)
vis2.set_title('Runtime(seconds) of different Models')

for index, row in result.iterrows():
    vis2.text(row.name,row.Timing, round(row.Timing,2), color='black', ha="center")

plt.ioff()<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup