Upload 5 files

model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8d5bd11c858325b31519dcad3a8ad8f69075f8edcfb269213705db7df8175e9
+size 277695407

testing.py

@@ -0,0 +1,69 @@
+import time
+
+print("Loading libraries...")
+start_time = time.time()
+
+import sklearn
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, mean_squared_error
+from sklearn.neural_network import MLPRegressor
+from sklearn.feature_extraction.text import CountVectorizer
+import matplotlib.pyplot as plt
+import numpy as np
+import datasets
+import pickle
+
+print(f"Libraries loaded in {round((time.time() - start_time) * 1000, 3)} ms.")
+
+print("Setting configuration...")
+start_time = time.time()
+
+# Set configuration
+sklearn.set_config(working_memory=4096)
+data_size = 100000
+
+
+print(f"Configuration set in {round((time.time() - start_time) * 1000, 3)} ms.")
+
+print("Loading model and vectorizer...")
+start_time = time.time()
+
+with open('model.pkl', 'rb') as model_file:
+    mlp = pickle.load(model_file)
+
+with open('vectorizer.pkl', 'rb') as vectorizer_file:
+    count_vect = pickle.load(vectorizer_file)
+
+print(f"Model and vectorizer loaded in {round((time.time() - start_time) * 1000, 3)} ms.")
+
+print("Loading data...")
+start_time = time.time()
+
+# Load data
+dataset = datasets.load_dataset('ucberkeley-dlab/measuring-hate-speech', 'binary')
+df = dataset['train'].to_pandas()
+
+print(f"Data loaded in {round((time.time() - start_time) * 1000, 3)} ms.")
+print(df.head())
+
+print("Fitting vectorizer...")
+start_time = time.time()
+
+# Extract text and labels
+X_text = df['text'][:data_size]  # Assuming 'text' is the column containing the text data
+
+# Convert text to vectors
+X = count_vect.fit(X_text)
+
+print(f"Vectorizer fit in {round((time.time() - start_time) * 1000, 3)} ms.")
+
+print("Predicting...")
+start_time = time.time()
+
+sentences = ["It is about time for all illegals to go back to their country of origin and keep our freeway open and prevent heavy traffic.", "OMG! The EGO\'s of these young, young, inexperienced women display are remarkable! So self absorbed - so, so proud of the mistakes their constituents all made in giving them a bit of power. They\'ll be quite surprised to find that the vast majority of Americans think that these women represent the very WORST of \"modern\" America. Egos out of control !!", "Instagram refugees lmao, let's build a wall instead"]
+vectorized_sentences = count_vect.transform(sentences)
+
+predictions = mlp.predict(vectorized_sentences)
+# write dict of sentences and predictions
+values = {sentences[i]: predictions[i] for i in range(len(sentences))}
+print(values)

train.py

@@ -0,0 +1,101 @@
+import time
+
+print("Loading libraries...")
+start_time = time.time()
+
+import sklearn
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, mean_squared_error
+from sklearn.neural_network import MLPRegressor
+from sklearn.feature_extraction.text import CountVectorizer
+import matplotlib.pyplot as plt
+import datasets
+import pickle
+
+print(f"Libraries loaded in {round((time.time() - start_time) * 1000, 3)} ms.")
+print("Loading vectorizer...")
+start_time = time.time()
+
+count_vect = CountVectorizer()
+
+print(f"Vectorizer loaded in {round((time.time() - start_time) * 1000, 3)} ms.")
+print(f"Saving vectorizer...")
+start_time = time.time()
+
+# Save vectorizer
+pickle.dump(count_vect, open('vectorizer.pkl', 'wb'))
+
+
+print("Setting configuration...")
+start_time = time.time()
+
+# Set configuration
+sklearn.set_config(working_memory=4096)
+data_size = 100000
+
+
+print(f"Configuration set in {round((time.time() - start_time) * 1000, 3)} ms.")
+print("Loading data...")
+start_time = time.time()
+
+# Load data
+dataset = datasets.load_dataset('ucberkeley-dlab/measuring-hate-speech', 'binary')
+df = dataset['train'].to_pandas()
+
+print(f"Data loaded in {round((time.time() - start_time) * 1000, 3)} ms.")
+print(df.head())
+
+print("Preprocessing data...")
+start_time = time.time()
+
+# Extract text and labels
+X_text = df['text'][:data_size]  # Assuming 'text' is the column containing the text data
+y_columns = ['hate_speech_score', 'sentiment', 'respect', 'insult', 'humiliate', 'status', 'dehumanize', 'violence', 'genocide', 'attack_defend', 'hatespeech']
+y = df[y_columns][:data_size]
+y = y.fillna(0)
+
+# Convert text to vectors
+X = count_vect.fit_transform(X_text)
+
+print(f"Data preprocessed in {round((time.time() - start_time) * 1000, 3)} ms.")
+print("Splitting data...")
+start_time = time.time()
+# Load data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)
+
+print(f"Data split in {round((time.time() - start_time) * 1000, 3)} ms.")
+print("Training model...")
+start_time = time.time()
+
+# Create MLPRegressor model
+mlp = MLPRegressor(hidden_layer_sizes=(256, 128, 64, 32, 16), activation='relu', max_iter=100, alpha=0.0001, learning_rate_init=0.003, solver='adam', verbose=True, tol=0.000000000001, early_stopping=False, n_iter_no_change=5000)
+mlp.fit(X_train, y_train)
+
+print(f"Model trained in {round((time.time() - start_time), 3)} s.")
+print("Evaluating model...")
+
+# Predict and score
+predictions = mlp.predict(X_test)
+print("Mean squared error: ", mean_squared_error(y_test, predictions))
+
+# Plot the loss curve
+plt.plot(mlp.loss_curve_)
+plt.title("Loss curve")
+plt.xlabel("Iteration")
+plt.ylabel("Loss")
+plt.show()
+
+print("Done!")
+
+# Save the model to disk
+
+filename = 'model.pkl'
+pickle.dump(mlp, open(filename, 'wb'))
+
+# Test the model for fun :)
+sentences = count_vect.fit_transform(["Fuck you you stupid nigger", "You're a piece of shit", "Awesome!", "Oh my god, I never realized that!"])
+
+predictions = mlp.predict(sentences)
+# Write dict of sentences and predictions
+values = {sentences[i]: predictions[i] for i in range(len(sentences))}
+

vectorizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fef08e75c9f3f95fffc3adcac5b216d5cba29accadff7f1bbd02b71e98cf0fae
+size 400