# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
# importing the preprocessed dataset

import json

jsonFile=json.load(open("preprocessed_dataset.json"))
class_label=["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]

clean_text= jsonFile["clean_text"]
result= jsonFile["result"]

for i in clean_text:
    i+= [-1]* (2041- len(i))

# +
# mapreduce-py installation (for using mapreduce within python)

# !pip install mapreduce-py

# +
# mapping text to numerical values

# def map_text():
x_dict={}
y_dict={}
hash_val={}

x_val=0
y_val=0

X = []
Y = []

# mapping and transforming text to numerical values
for i in range(len(clean_text)):
    store=[]
    for j in clean_text[i]:
        if j in x_dict:
            store.append(x_dict[j])
        else:
            x_dict[j]=x_val
            x_val+=1
            store.append(x_dict[j])
    X.append(store)

    s=''.join([str(x) for x in result[i]])
    if s in y_dict:
        Y.append(y_dict[s])
    else:
        y_dict[s]=y_val
        hash_val[y_val]=s
        y_val+=1
        Y.append(y_dict[s])
#     return X, Y

# +
from sklearn import tree

def train(X,Y):
    arr=[]
    for i in X:
       i+= [-1]* (2041- len(i))

    from sklearn import tree
    clf = tree.DecisionTreeClassifier()
    clf = clf.fit(X, Y)
    return clf



# +
def test(X, clf):
    x=clf.predict(X)
    test_set= X[0:int((0.2)*len(X))]
    x=clf.predict(X)

    labels = result[0:int((0.2)*len(X))]
    predictions = x
    from sklearn.metrics import precision_score

#     p=precision_score(labels, x, average='micro')
#     r=recall_score(labels, x, average='micro')
#     a=accuracy_score(labels, x, average='micro')
    
    return predictions
    


# +
# training and testing decision tree

from mapreduce import *

MR=MapReducer().prefilter(map_text).mapper(train).reducer(test, None)
res = MR(range(len(MR)))
print('MR Result  :', res)

accuracy, precision, recall=res
print("accuracy:", accuracy)
print("precision:", precision)
print("recall:", recall)

print("F1 Score:", ((2*precision*recall)/(precision+recall)))
# -

# svm 
import numpy as np
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
def train():
    from sklearn.svm import SVC
    clf = make_pipeline(StandardScaler(), SVC(gamma='auto'))
    clf.fit(X, Y)
    return clf



# +
# training and testing svm

from mapreduce import *

MR=MapReducer().prefilter(map_text).mapper(train).reducer(test, None)
res = MR(range(len(MR)))

accuracy, precision, recall=res
print("accuracy:", accuracy)
print("precision:", precision)
print("recall:", recall)

print("F1 Score:", ((2*precision*recall)/(precision+recall)))

# +
# prediction with sample reddit data

# fetching reddit data
import http.client

conn = http.client.HTTPSConnection("reddit3.p.rapidapi.com")

headers = {
    'x-rapidapi-host': "reddit3.p.rapidapi.com",
    'x-rapidapi-key': "52e70ee3a0mshe32a5a1bbaa67bcp1e2257jsndfce50b84406"
    }

conn.request("GET", "/all?url=https%3A%2F%2Fwww.reddit.com%2Fr%all", headers=headers)

res = conn.getresponse()
data = res.read()

text_data=data["replies"]
text_data=text_data[0:min(20, len(text_data))]


# predicting

res= test(text_data, train(X,Y))
print(res)

# +
# printing hash values of the results

for i in res:
    print(hash_val[i])
    r=""
    for j in hash_val[i]:
        r+=class_label[int(j)]
        r+=' '
    print(r)
        

StarcoderdataJupyterScriptsDedupFiltered