# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
from sklearn.neural_network import MLPClassifier
from sklearn import preprocessing as pre
#from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import ShuffleSplit
from sklearn.externals import joblib

# Preprocessing dataset comments hate wikipedia
df = pd.read_csv('../MachineLearning/DATASETS/Identificador de comentários toxicos/train.csv/train.csv')
df['is_toxic'] = df['identity_hate'] + df['insult'] + df['obscene'] + df['severe_toxic'] + df['threat'] + df['toxic']
df.drop(['identity_hate', 'insult', 'obscene', 'severe_toxic', 'threat', 'toxic', 'id'], axis=1, inplace=True)
df['is_toxic'] = df['is_toxic'].apply(lambda x: 0 if x == 0 else 1)
df.head()

# Preprocessing dataset comments hate twitter
df2 = pd.read_csv('../NLP/labeled_data.csv')
df2.drop(['Unnamed: 0', 'count', 'hate_speech', 'offensive_language', 'neither'], axis=1, inplace=True)
df2['class'] = df2['class'].apply(lambda x: 0 if x == 2 else 1)
df2['comment_text'] = df2['tweet']
df2['is_toxic'] = df2['class']
df2.drop(['class', 'tweet'], axis=1, inplace=True)
df2.head()

# Join datasets
df3 = pd.concat([df,df2], axis=0)
df3.head()

print(f'Number of hate comments: {len(df3[df3["is_toxic"] == 1])}')
print(f'Number of non-hateful comments: {len(df3[df3["is_toxic"] == 0])}')

# +
# Extracting features from text using TFiDF
X = df['comment_text']
y = df['is_toxic']

vectorizer = TfidfVectorizer(stop_words='english')
X_tfidf = vectorizer.fit_transform(X)
# -

with open('tfidf_vectorizer.pkl', 'wb') as fin:
    joblib.dump(vectorizer, fin)

print(vectorizer.get_stop_words())

vectorizer.vocabulary_

print(vectorizer.idf_)

text_clf =  MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(5, 2), random_state=1)
cv = ShuffleSplit(n_splits=10, test_size=0.3, random_state=1)
scores = cross_val_score(text_clf, X_tfidf, y, cv=cv, scoring='f1_macro')
print(f'F1 Score: {scores.mean()} (+/- {scores.std() * 2})')

joblib.dump(text_clf, 'model_mlp_cls.pkl')

# !pip install googletrans

from googletrans import Translator
translator = Translator()
translator.translate('<NAME>')
# <Translated src=ko dest=en text=Good evening. pronunciation=Good evening.>
trans = translator.translate('<NAME>', dest='en')
trans.text

detect = translator.detect('이 문장은 한글로 쓰여졌습니다.')
detect.lang

StarcoderdataJupyterScriptsDedupFiltered