# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/Paul-mwaura/CrowdsourcingML/blob/main/CrowdsourcingML_Notebook.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="5MDIgdczdwuk"
# # CrowdsourcingML on Amazon Data

# + colab={"base_uri": "https://localhost:8080/"} id="4mtM_XSvOEgC" outputId="4b526f83-c06c-4883-831c-302bad4ea633"
pip install scikit-multilearn

# + [markdown] id="-Ta3RXBFd6l7"
# ## Import Necessary Libraries

# + id="XFwsO5ATMYWq"
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
# %matplotlib inline

from sklearn import model_selection
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# Machine Learning Algorithms
from skmultilearn.problem_transform import ClassifierChain
from sklearn.naive_bayes import GaussianNB
from sklearn.cluster import KMeans
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import BernoulliNB

# Metric Libraries
from sklearn.metrics import accuracy_score
from sklearn import metrics

 
import warnings
warnings.filterwarnings("ignore")

# + [markdown] id="OdsKO8PUd3jI"
# ## Loading the Dataset

# + colab={"base_uri": "https://localhost:8080/", "height": 202} id="bqgKCoe-MjNj" outputId="74726c9e-c8f6-4db9-83d8-94b6d1d2f90e"
df = pd.read_csv("amazon.csv")
df.head()

# + [markdown] id="Vyj1Lb9oMG_S"
# **Rename the Columns**

# + colab={"base_uri": "https://localhost:8080/", "height": 110} id="2fZs8YxMMG0P" outputId="83f74ad2-b698-4843-f361-6356f676b47b"
df.columns = ['worker_id', 'task_id', 'worker_reviews', 'expert_reviews', 'time_taken']
df.head(2)

# + [markdown] id="EG8gsv2VeH7W"
# **Check Data Types**

# + colab={"base_uri": "https://localhost:8080/"} id="lzu87I9yM5eb" outputId="e506f352-91a8-4d66-9ae6-006975e34aed"
df.info()

# + [markdown] id="KWOTocm8eNEj"
# **Get a Description of float and integer variables**

# + colab={"base_uri": "https://localhost:8080/", "height": 171} id="CUatv7EdNB4L" outputId="b922de10-4157-468d-dd7f-e184eff23cce"
df.describe().T

# + [markdown] id="c2gK4517eUj7"
# **Check the size of the dataset**

# + colab={"base_uri": "https://localhost:8080/"} id="wq_2lHCPNO68" outputId="883201aa-9528-4bea-b498-0e6abe56cf93"
df.shape

# + [markdown] id="NMP6LAlneacF"
# **Print dataset column names**

# + colab={"base_uri": "https://localhost:8080/"} id="L2hFGNCdNS9h" outputId="9aea1a4c-4655-447f-cb60-401381832392"
columns = df.columns
columns

# + [markdown] id="zESzW7Z3ee24"
# **Get the count of unique values in the columns**

# + colab={"base_uri": "https://localhost:8080/"} id="jQEkE9AHOuAk" outputId="a1311102-9de5-4297-a084-e5173954339c"
for col in columns:
  print(f'Length of Unique values in {col} is: {len(df[col].unique())}')

# + [markdown] id="FaxLv_cSdDJY"
# ## Data Cleaning

# + [markdown] id="dGeDlHsbemzo"
# **Check for null values**

# + colab={"base_uri": "https://localhost:8080/"} id="F5_Z1n8cPDWW" outputId="4028c0c2-ae6a-4e9b-83df-60a9297b7946"
df.isna().sum()

# + [markdown] id="gvwYGB-pep6_"
# **Check for Duplicated Values**

# + colab={"base_uri": "https://localhost:8080/"} id="d1KUhe-sc-jd" outputId="c7a6e63c-fa0a-492b-b472-2ddf33bdf02d"
df.duplicated().sum()

# + [markdown] id="h8m5b8JYet3a"
# **Get dummy values for the worker_id column**

# + colab={"base_uri": "https://localhost:8080/", "height": 110} id="9xBJdcVBOVk1" outputId="53935859-7369-45e9-d403-4e51439467aa"
df['worker_id'] = pd.get_dummies(df['worker_id'])
df.head(2)

# + [markdown] id="3wHT1FzMdNAv"
# ## Exploratory Data Analysis

# + [markdown] id="Rsuxb7hllEqT"
# ### Scatter Plots

# + [markdown] id="NXJjLF59lIp3"
# #### Worker Reviews

# + colab={"base_uri": "https://localhost:8080/", "height": 351} id="AEu_iAewdRWb" outputId="14144e85-aef0-4bb7-cf73-249101c9cfa5"
sns.lmplot(data=df, x="time_taken", y="task_id", col="worker_reviews", hue="worker_reviews")

# + [markdown] id="huKFtwh4lOCw"
# #### Expert Reviews

# + colab={"base_uri": "https://localhost:8080/", "height": 351} id="YkaAbEigkiB0" outputId="3e2fa6c2-d1b5-49df-ce9c-0fff759dd160"
sns.lmplot(data=df, x="time_taken", y="task_id", col="expert_reviews", hue="expert_reviews")

# + [markdown] id="QPV4K8CaE-1y"
# ###  Joint Plots

# + [markdown] id="ilnPJdGAFCYM"
# #### Worker Reviews

# + colab={"base_uri": "https://localhost:8080/", "height": 459} id="3z-0O5z4EZAX" outputId="d217b977-f07e-4a73-d74e-29a69422df7b"
sns.jointplot(data=df, x="time_taken", y="task_id", hue="worker_reviews")
plt.title('Time Taken and Task ID Joint Plot for Worker Reviews')

# + [markdown] id="CZUAtFEeFDux"
# #### Expert Reviews

# + colab={"base_uri": "https://localhost:8080/", "height": 459} id="3Cvh4MbmDXKG" outputId="daabb8b0-4a61-40a3-c155-19287fc1af07"
sns.jointplot(data=df, x="time_taken", y="task_id", hue="expert_reviews")
plt.title('Time Taken and Task ID Joint Plot for Expert Reviews')

# + [markdown] id="90l1uNVTL8x2"
# ## Implementing the solution

# + [markdown] id="qU_sT7Xne4pU"
# **Split data into x(features) and y(labels)**

# + colab={"base_uri": "https://localhost:8080/", "height": 110} id="UXy4Pk5sMBDa" outputId="87e093e5-8360-41b2-af1f-e03ac185db3d"
x = df[['task_id', 'time_taken']]
y = df[['worker_reviews', 'expert_reviews']]
y.head(2)

# + [markdown] id="QDSLR7cmfCHr"
# **Split data into train(80%)and test(20%)**

# + id="7egTlT6jNrld"
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=12)

# + [markdown] id="MhcUY3YaWiIP"
# ### Classifier Chains
#

# + [markdown] id="58oZwffvF57I"
# <a href='https://scikit-learn.org/stable/modules/generated/sklearn.multioutput.ClassifierChain.html#sklearn.multioutput.ClassifierChain'>Classifier chains</a> is a machine learning method for problem transformation in multi-label classification. It combines the computational efficiency of the Binary Relevance method while still being able to take the label dependencies into account for classification.
# >>
# Each model makes a prediction in the order specified by the chain using all of the available features provided to the model plus the predictions of models that are earlier in the chain.
# >>
# When predicting, the true labels will not be available. Instead the predictions of each model are passed on to the subsequent models in the chain to be used as features.
# >>
# Clearly the order of the chain is important. The first model in the chain has no information about the other labels while the last model in the chain has features indicating the presence of all of the other labels. In general one does not know the optimal ordering of the models in the chain so typically many randomly ordered chains are fit and their predictions are averaged together.

# + [markdown] id="MqyrJLxwZyMc"
# #### GaussianNB Classifier

# + colab={"base_uri": "https://localhost:8080/"} id="3Qss51aWL8kg" outputId="3515a85c-a10a-41f8-f7f7-f04ace125657"
# using classifier chains

# initialize classifier chains multi-label classifier
# with a gaussian naive bayes base classifier
gaussian = GaussianNB()
gaussian_clf = ClassifierChain(gaussian)

sc = StandardScaler()

X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
# train
gaussian_clf.fit(X_train, y_train)

# predict
gaussian_preds = gaussian_clf.predict(X_test)

print(f'Gaussian accuracy score: {accuracy_score(y_test,gaussian_preds)*100}%')

# + colab={"base_uri": "https://localhost:8080/", "height": 202} id="JSoIU2R9N3X0" outputId="133ebcfa-d17b-42df-adc5-222c6fce89cb"
gaussian_preds = pd.DataFrame.sparse.from_spmatrix(gaussian_preds)
gaussian_preds.columns=['worker', 'expert']

gaussian_preds.head()

# + colab={"base_uri": "https://localhost:8080/"} id="QbQ9ePu5gB8Q" outputId="7a78a1e7-4cb4-41a0-b1cf-04446e03ad88"
gaussian_pred_w = gaussian_preds['worker']
y_test_gw = y_test['worker_reviews']
gaussian_pred_e = gaussian_preds['expert']
y_test_ge = y_test['expert_reviews']

print(f"Worker Precision:, {metrics.precision_score(y_test_gw, gaussian_pred_w)*100}")
print(f"Worker Recall:, {metrics.recall_score(y_test_gw, gaussian_pred_w)*100}\n")
print(f"Expert Precision:, {metrics.precision_score(y_test_ge, gaussian_pred_e)*100}")
print(f"Expert Recall:, {metrics.recall_score(y_test_ge, gaussian_pred_e)*100}")

# + [markdown] id="Eip1W1hliZ6c"
# **Confusion Matrix for Workers**

# + colab={"base_uri": "https://localhost:8080/", "height": 421} id="bAmDC07QhGJB" outputId="8ab6acd3-1896-4ed4-f04d-de8c2d12fa9f"
cm = pd.crosstab(y_test_gw, gaussian_pred_w, rownames=['Actual'], colnames=['Predicted'])
print(cm)
fig, (ax1) = plt.subplots(ncols=1, figsize=(5,5))
sns.heatmap(cm, 
            xticklabels=['negative', 'positive'],
            yticklabels=['negative', 'positive'],
            annot=True,ax=ax1,
            linewidths=.2,linecolor="Darkblue", cmap="Blues")
plt.title('Confusion Matrix for Workers.', fontsize=14)
plt.show()
# 1= default
# 0 = No default

# + [markdown] id="F8qzfXfGiVxA"
# **Confusion Matrix for Experts**

# + colab={"base_uri": "https://localhost:8080/", "height": 421} id="x3LHZOJHhoJa" outputId="20a148fe-0a00-4d07-b49e-545b6a4667ad"
cm = pd.crosstab(y_test_ge, gaussian_pred_e, rownames=['Actual'], colnames=['Predicted'])
print(cm)
fig, (ax1) = plt.subplots(ncols=1, figsize=(5,5))
sns.heatmap(cm, 
            xticklabels=['negative', 'positive'],
            yticklabels=['negative', 'positive'],
            annot=True,ax=ax1,
            linewidths=.2,linecolor="Darkblue", cmap="Blues")
plt.title('Confusion Matrix for Experts.', fontsize=14)
plt.show()
# 1= default
# 0 = No default

# + [markdown] id="DvVuryG3Z3tK"
# #### Logistic Regression

# + colab={"base_uri": "https://localhost:8080/"} id="M0XipFcdQxuy" outputId="218d7ac1-0d8e-4be6-d440-c5b55c65eb88"
#Logistic Regression

log_reg = LogisticRegression()
lr_clf = ClassifierChain(log_reg)

X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)

# train
lr_clf.fit(X_train, y_train)

# predict
log_reg_preds = lr_clf.predict(X_test)

from sklearn import metrics

print(f'Accuracy: {accuracy_score(y_test,log_reg_preds)*100}%')

# + colab={"base_uri": "https://localhost:8080/", "height": 110} id="atiBnj2VaLaF" outputId="11569021-2a7e-4e8f-e7aa-a2edffd4139f"
log_reg_preds = pd.DataFrame.sparse.from_spmatrix(log_reg_preds)
log_reg_preds.columns=['worker_reviews', 'expert_reviews']
log_reg_preds.head(2)

# + colab={"base_uri": "https://localhost:8080/"} id="-_mJV1TKs-sd" outputId="603e22a7-23ac-4204-d4f8-8c0584d6d8c9"
log_reg_pred_w = log_reg_preds['worker_reviews']
y_test_lw = y_test['worker_reviews']
log_reg_pred_e = log_reg_preds['expert_reviews']
y_test_le = y_test['expert_reviews']

print(f"Worker Precision:, {metrics.precision_score(y_test_lw, log_reg_pred_w)*100}")
print(f"Worker Recall:, {metrics.recall_score(y_test_lw, log_reg_pred_w)*100}\n")
print(f"Expert Precision:, {metrics.precision_score(y_test_le, log_reg_pred_e)*100}")
print(f"Expert Recall:, {metrics.recall_score(y_test_le, log_reg_pred_e)*100}")

# + [markdown] id="qkzzQv3IigBK"
# **Confusion Matrix for Workers**

# + colab={"base_uri": "https://localhost:8080/", "height": 421} id="RhJ0kKyhfS6q" outputId="85a4b627-1d2e-4083-b4dd-51e10eedbe40"
cm = pd.crosstab(y_test_lw, log_reg_pred_w, rownames=['Actual'], colnames=['Predicted'])
print(cm)
fig, (ax1) = plt.subplots(ncols=1, figsize=(5,5))
sns.heatmap(cm, 
            xticklabels=['negative', 'positive'],
            yticklabels=['negative', 'positive'],
            annot=True,ax=ax1,
            linewidths=.2,linecolor="Darkblue", cmap="Blues")
plt.title('Confusion Matrix for Workers.', fontsize=14)
plt.show()
# 1= default
# 0 = No default

# + [markdown] id="CVrIEBsIiksY"
# **Confusion Matrix for Experts**

# + colab={"base_uri": "https://localhost:8080/", "height": 421} id="aNd8cwTohyDK" outputId="01825309-79f5-4c4f-f1af-731d2ed0b7a3"
cm = pd.crosstab(y_test_le, log_reg_pred_e, rownames=['Actual'], colnames=['Predicted'])
print(cm)
fig, (ax1) = plt.subplots(ncols=1, figsize=(5,5))
sns.heatmap(cm, 
            xticklabels=['negative', 'positive'],
            yticklabels=['negative', 'positive'],
            annot=True,ax=ax1,
            linewidths=.2,linecolor="Darkblue", cmap="Blues")
plt.title('Confusion Matrix for Experts.', fontsize=14)
plt.show()
# 1= default
# 0 = No default

# + [markdown] id="SiOipcSqrrRP"
# #### KMeans Classifier

# + colab={"base_uri": "https://localhost:8080/"} id="_Ry61mX2rym4" outputId="66d4cf83-6e60-48b4-b640-51aac905fdb7"
#KMeans

k_means = KMeans(n_clusters=2)
kmeans_clf = ClassifierChain(k_means)

# train
kmeans_clf.fit(X_train, y_train)

# predict
kmeans_preds = kmeans_clf.predict(X_test)

print(f'KMeans accuracy score: {accuracy_score(y_test,kmeans_preds)*100}%')

# + colab={"base_uri": "https://localhost:8080/", "height": 141} id="mk4c9kcnsETs" outputId="f3111eff-5f31-4171-c433-f2420f53441d"
kmeans_preds = pd.DataFrame.sparse.from_spmatrix(kmeans_preds)
kmeans_preds.columns=['worker_reviews', 'expert_reviews']
kmeans_preds.head(3)

# + colab={"base_uri": "https://localhost:8080/"} id="s584fUhL_a8t" outputId="a213299d-4d6e-403f-9025-6fd7b977e9ea"
kmeans_pred_w = kmeans_preds['worker_reviews']
y_test_kmw = y_test['worker_reviews']
kmeans_pred_e = kmeans_preds['expert_reviews']
y_test_kme = y_test['expert_reviews']

print(f"Worker Precision:, {metrics.precision_score(y_test_kmw, kmeans_pred_w)*100}")
print(f"Worker Recall:, {metrics.recall_score(y_test_kmw, kmeans_pred_w)*100}\n")
print(f"Expert Precision:, {metrics.precision_score(y_test_kme, kmeans_pred_e)*100}")
print(f"Expert Recall:, {metrics.recall_score(y_test_kme, kmeans_pred_e)*100}")

# + colab={"base_uri": "https://localhost:8080/", "height": 421} id="chockq6WBp_D" outputId="827a0c73-a6e9-4e99-bb31-c0e7b14b2786"
cm = pd.crosstab(y_test_kmw, kmeans_pred_w, rownames=['Actual'], colnames=['Predicted'])
print(cm)
fig, (ax1) = plt.subplots(ncols=1, figsize=(5,5))
sns.heatmap(cm, 
            xticklabels=['negative', 'positive'],
            yticklabels=['negative', 'positive'],
            annot=True,ax=ax1,
            linewidths=.2,linecolor="Darkblue", cmap="Blues")
plt.title('Confusion Matrix for Worker.', fontsize=14)
plt.show()
# 1= default
# 0 = No default

# + colab={"base_uri": "https://localhost:8080/", "height": 421} id="qrwd2i8_Aa1P" outputId="05455675-cf27-45e9-bb5c-a04e684f856c"
cm = pd.crosstab(y_test_kme, kmeans_pred_e, rownames=['Actual'], colnames=['Predicted'])
print(cm)
fig, (ax1) = plt.subplots(ncols=1, figsize=(5,5))
sns.heatmap(cm, 
            xticklabels=['negative', 'positive'],
            yticklabels=['negative', 'positive'],
            annot=True,ax=ax1,
            linewidths=.2,linecolor="Darkblue", cmap="Blues")
plt.title('Confusion Matrix for Experts.', fontsize=14)
plt.show()
# 1= default
# 0 = No default

# + [markdown] id="fdZC8UOsdX_t"
# #### BernouliNB Classifier

# + colab={"base_uri": "https://localhost:8080/"} id="qhJ5YFzRsvvO" outputId="b6e91576-cf1f-43b8-aa1e-236d52d6ce16"
#Naive Bayes(Bernouli)

bernNB = BernoulliNB(binarize = 0.1)
bernNB_clf = ClassifierChain(bernNB)

# train
bernNB_clf.fit(X_train, y_train)

# predict
bernNB_preds = bernNB_clf.predict(X_test)

print(f'Bernouli accuracy score: {accuracy_score(y_test,bernNB_preds)*100}%')

# + colab={"base_uri": "https://localhost:8080/", "height": 141} id="BBIyhCcxy0l7" outputId="7d3d3b71-3718-4822-8294-89e1ab379b6e"
bernNB_preds = pd.DataFrame.sparse.from_spmatrix(bernNB_preds)
bernNB_preds.columns=['worker_reviews', 'expert_reviews']
bernNB_preds.head(3)

# + [markdown] id="BdxoBmR6a9eD"
# #### 5-fold cross validation

# + colab={"base_uri": "https://localhost:8080/"} id="1eDOq-XHawtx" outputId="9926408e-7812-48b6-dd92-144153bfccde"
print('5-fold cross validation: \n')

labels = ['Gaussian', 'Logistic Regression', 'K Means', 'Naive Bayes']


for clf, label in zip([gaussian_clf, lr_clf, kmeans_clf, bernNB_clf], labels):
  scores = model_selection.cross_val_score(clf, x, y, cv=5, scoring='accuracy')
  print('Accuracy: %0.2f (+/- %0.2f) [%s]' %(scores.mean()*100, scores.std(), label))

value_hard = 0.82

# + [markdown] id="xQnu8A-8tra6"
# Now we can proceed to identify bias using our algorithms.

# + [markdown] id="_nQ3fUANdf__"
# ## Bias detection and mitigation

# + [markdown] id="W3_iVPQrFF0-"
# ### Install aif360

# + colab={"base_uri": "https://localhost:8080/"} id="5R8gnC44d2FW" outputId="e200d668-d0a4-429c-a561-9dc250e91403"
pip install aif360[all]

# + [markdown] id="LViXgw1iFMIP"
# ### Identifying Bias in the Data

# + colab={"base_uri": "https://localhost:8080/"} id="r0eNS2Rher9z" outputId="0b683e27-ccab-4b77-c7ef-ab6f4ab1a334"
positive_df = df[df['worker_reviews'] == 1]
num_of_privileged = len(positive_df)

negative_df = df[df['worker_reviews'] == 0]
num_of_unprivileged = len(negative_df)

print(f'Num privileged: {num_of_privileged}')
print(f'Num unprivileged: {num_of_unprivileged}')

# + colab={"base_uri": "https://localhost:8080/"} id="CMF-s5zqphT1" outputId="3edaa822-6d31-44b9-f82e-05ef9f5a451c"
unprivileged_outcomes = negative_df[negative_df['expert_reviews'] == 1].shape[0]
unprivileged_ratio = unprivileged_outcomes/num_of_unprivileged
print(f'Unprivileged ratio: {unprivileged_ratio}')

# + colab={"base_uri": "https://localhost:8080/"} id="Wk2hhCWLpoay" outputId="308ab206-ba7b-4477-cb78-8f0790e44850"
privileged_outcomes = positive_df[positive_df['expert_reviews'] == 1].shape[0]
privileged_ratio = privileged_outcomes/num_of_privileged
print(f'Privileged ratio: {privileged_ratio}')

# + colab={"base_uri": "https://localhost:8080/"} id="5gqmwMOtrPIE" outputId="2d43f036-1a62-4aa5-90e0-a6da2840dadf"
# Calculating disparate impact
disparate_impact = unprivileged_ratio / privileged_ratio
print(f'Disparate Impact: {disparate_impact}')

# + [markdown] id="uV0E56BFuJyL"
# The industry standard is a four-fifths rule: if the unprivileged group receives a positive outcome less than 80% of their proportion of the privilege group, this is a disparate impact violation. However, you may decide to increase this for your business.
#
# In this scenario, we are below the threshold of 0.98 so we deem this to be fair.
#
# A disparate income ratio of 1 indicates complete equality.

# + [markdown] id="7usEoTaZu-BG"
# ### Mitigating Bias with AI Fairness 360

# + id="IPx3y-izrzM_"
import aif360
from aif360.algorithms.preprocessing import DisparateImpactRemover
binaryLabelDataset = aif360.datasets.BinaryLabelDataset(
    favorable_label=1,
    unfavorable_label=0,
    df=df,
    label_names=['expert_reviews'],
    protected_attribute_names=['worker_reviews'])

# + colab={"base_uri": "https://localhost:8080/", "height": 222} id="bw0pheT2y0Ha" outputId="f93508f1-2482-4c14-d5f3-011d0d477065"
di = DisparateImpactRemover(repair_level = 1.0)
dataset_transf_train = di.fit_transform(binaryLabelDataset)
transformed = dataset_transf_train.convert_to_dataframe()[0]
transformed.describe().T

# + id="h_NYvOIO2t74"
x_trans = transformed[['task_id', 'time_taken']]
y = transformed[['worker_reviews', 'expert_reviews']]
scaler = StandardScaler()
x_trans = scaler.fit_transform(x_trans)
x_trans_train,x_trans_test,y_trans_train,y_trans_test = train_test_split(x_trans, y, test_size=0.2, random_state = 0)

# + [markdown] id="__jb6UpkFYFT"
# ### Models

# + [markdown] id="BNZkaDKkB4Jq"
# #### GaussianNB Classifier

# + colab={"base_uri": "https://localhost:8080/"} id="myVtAPPqB37l" outputId="b69d1978-ded8-4d3a-cd83-24f12b78d16b"
gaussian_clf.fit(x_trans_train, y_trans_train)

y_trans_preds_g = gaussian_clf.predict(x_trans_test)


print(f'Gaussian accuracy score: {accuracy_score(y_trans_test, y_trans_preds_g)*100}%\n')

# Convert predictions from sparse matrix to dataframe.
y_trans_preds_g = pd.DataFrame.sparse.from_spmatrix(y_trans_preds_g)
y_trans_preds_g.columns=['worker_reviews', 'expert_reviews']

# Split the labels into two. (wokers and experts)
gaussian_trans_pred_w = y_trans_preds_g['worker_reviews']
y_trans_test_gw = y_trans_test['worker_reviews']
gaussian_trans_pred_e = y_trans_preds_g['expert_reviews']
y_trans_test_ge = y_test['expert_reviews']

print(f"Worker Precision:, {metrics.precision_score(y_trans_test_gw, gaussian_trans_pred_w)*100}")
print(f"Worker Recall:, {metrics.recall_score(y_trans_test_gw, gaussian_trans_pred_w)*100}\n")
print(f"Expert Precision:, {metrics.precision_score(y_trans_test_gw, gaussian_trans_pred_e)*100}")
print(f"Expert Recall:, {metrics.recall_score(y_trans_test_gw, gaussian_trans_pred_e)*100}")

# + [markdown] id="ouJSfwlIB0jQ"
# #### Logistic Regression

# + colab={"base_uri": "https://localhost:8080/"} id="wEClleKj218Y" outputId="d710bf07-20c4-4f17-925d-6767916303b7"
lr_clf.fit(x_trans_train, y_trans_train)

y_trans_preds_lr = lr_clf.predict(x_trans_test)

print(f'Logistic accuracy score: {accuracy_score(y_trans_test, y_trans_preds_lr)*100}%\n')

# Convert predictions from sparse matrix to dataframe.
y_trans_preds_lr = pd.DataFrame.sparse.from_spmatrix(y_trans_preds_lr)
y_trans_preds_lr.columns=['worker_reviews', 'expert_reviews']

# Split the labels into two. (wokers and experts)
lr_trans_pred_w = y_trans_preds_lr['worker_reviews']
y_trans_test_lw = y_trans_test['worker_reviews']
lr_trans_pred_e = y_trans_preds_lr['expert_reviews']
y_trans_test_le = y_trans_test['expert_reviews']

print(f"Worker Precision:, {metrics.precision_score(y_trans_test_lw, lr_trans_pred_w)*100}")
print(f"Worker Recall:, {metrics.recall_score(y_trans_test_lw, lr_trans_pred_w)*100}\n")
print(f"Expert Precision:, {metrics.precision_score(y_trans_test_lw, lr_trans_pred_e)*100}")
print(f"Expert Recall:, {metrics.recall_score(y_trans_test_lw, lr_trans_pred_e)*100}")

# + [markdown] id="hwGsUPaoEf6J"
# #### KMeans Classifier

# + colab={"base_uri": "https://localhost:8080/"} id="4jrxLj5KEfeS" outputId="692029d6-6e6d-4c58-c3d6-1c69be406852"
# train
kmeans_clf.fit(x_trans_train, y_trans_train)

# predict
kmeans_trans_preds = kmeans_clf.predict(x_trans_test)

print(f'KMeans accuracy score: {accuracy_score(y_trans_test,kmeans_trans_preds)*100}%\n')

# Convert predictions from sparse matrix to dataframe.
kmeans_trans_preds = pd.DataFrame.sparse.from_spmatrix(kmeans_trans_preds)
kmeans_trans_preds.columns=['worker_reviews', 'expert_reviews']

# Split the labels into two. (wokers and experts)
kmeans_trans_pred_w = kmeans_trans_preds['worker_reviews']
y_trans_test_kw = y_trans_test['worker_reviews']
kmeans_trans_pred_e = kmeans_trans_preds['expert_reviews']
y_trans_test_ke = y_trans_test['expert_reviews']

print(f"Worker Precision:, {metrics.precision_score(y_trans_test_kw, kmeans_trans_pred_w)*100}")
print(f"Worker Recall:, {metrics.recall_score(y_trans_test_kw, kmeans_trans_pred_w)*100}\n")
print(f"Expert Precision:, {metrics.precision_score(y_trans_test_kw, kmeans_trans_pred_e)*100}")
print(f"Expert Recall:, {metrics.recall_score(y_trans_test_kw, kmeans_trans_pred_e)*100}")

# + [markdown] id="2g-OklFHCirT"
# #### Naive Bayes Classifier

# + colab={"base_uri": "https://localhost:8080/"} id="ojUyDCleChz2" outputId="fda00603-0814-4fc1-bcff-22cc8e80cd67"
# train
bernNB_clf.fit(x_trans_train, y_trans_train)

# predict
bernNB_trans_preds = bernNB_clf.predict(x_trans_test)

print(f'BernouliNB accuracy score: {accuracy_score(y_trans_test,bernNB_trans_preds)*100}%\n')

# Convert predictions from sparse matrix to dataframe.
bernNB_trans_preds = pd.DataFrame.sparse.from_spmatrix(bernNB_trans_preds)
bernNB_trans_preds.columns=['worker_reviews', 'expert_reviews']

# Split the labels into two. (wokers and experts)
bernNB_trans_pred_w = bernNB_trans_preds['worker_reviews']
y_trans_test_bern_w = y_trans_test['worker_reviews']
bernNB_trans_pred_e = bernNB_trans_preds['expert_reviews']
y_trans_test_bern_e = y_trans_test['expert_reviews']

print(f"Worker Precision:, {metrics.precision_score(y_trans_test_bern_w, bernNB_trans_pred_w)*100}")
print(f"Worker Recall:, {metrics.recall_score(y_trans_test_bern_w, bernNB_trans_pred_w)*100}\n")
print(f"Expert Precision:, {metrics.precision_score(y_trans_test_bern_w, bernNB_trans_pred_e)*100}")
print(f"Expert Recall:, {metrics.recall_score(y_trans_test_bern_w, bernNB_trans_pred_e)*100}")

StarcoderdataJupyterScriptsDedupFiltered