# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
import math
from sklearn.model_selection import train_test_split
from collections import Counter

trainset = pd.read_csv("E:\\programs\\python\\NLP\\trainset.csv", delimiter=',', encoding='utf-8')
validationset = pd.read_csv("E:\\programs\\python\\NLP\\validationset.csv", delimiter=',', encoding='utf-8')

# display complete contents of a dataframe without any kind of truncation
pd.set_option('display.max_rows',None)
pd.set_option('display.max_columns',None)
pd.set_option('display.width',None)
pd.set_option('display.max_colwidth',-1)

print(trainset.head(10)) # print 10 first rows

train_len = trainset['Unnamed: 0'].count() #the first row is the topics so we got 700000 data
valid_len = validationset['Unnamed: 0'].count()  #the first row is the topics so we got 147635 data

print(train_len) #printing the data rows in trainset
print(valid_len) #printing the data rows in validationset

number_per_cat = trainset.groupby("cat1")["id"].count() #calculating the number of items in each category

number_per_cat

arr = pd.read_csv("E:\\programs\\python\\NLP\\projects\\persian.csv", sep="\n", encoding='utf-8')
stop_words = arr.values.tolist() #converting the stop words into a list
stop_words = [item for sublist in stop_words for item in sublist] #converting stop words into a 1d list

descriptions = trainset['desc']
titles = trainset['title']

descriptions.head(3) #printing the first 3 rows of description

sample = descriptions.head(5) # first 5 rows of description


# +
def preprocessing(text):
    """
     This method takes an unprocessed text and removes its stop words, and punctuations. It takes a list of simple texts and 
     returns a list containing words that are processed for each text.
     text: the unprocessed text
     cleared_text: the processed text
    """
    processed_description = [] #the processed descriptions would go here
    cleared_text = [] #a list to save all of the processed descriptions
    for i in range(len(text)):
        desc = text[i]
        words = desc.split(" ")
        
#     print(len(words))
        for j in range(len(words)):
            if words[j] not in stop_words:
                processed_description.append(words[j])
#     print(words)
#     print("======================================")
#     print(len(processed_description))
#         print(processed_description)
        cleared_text.append(processed_description)
#         print(cleared_text)
        processed_description = []
    return cleared_text


# -

uncleared_text = [] #an example to show how the preprocessing works
for i in range(len(sample)):
    desc = sample[i]
    words = desc.split(" ")
    uncleared_text.append(words)
print(uncleared_text) #unprocessed text for the first five rows
print("========================")
cleared_text = preprocessing(sample)
print(cleared_text) #processed text for the first five rows

cleared_descriptions = preprocessing(descriptions.head(1000)) #preprocessed descriptions
cleared_titles = preprocessing(titles.head(1000)) #preprocessed titles

number_of_texts = len(cleared_titles) #total number of texts in our corpus
print(number_of_texts)


def tf_idf(text):
    """
     This method calculates the tf_idf which is tf(word, text) * idf(word). With tf_idf we represent texts in numerical forms
     text: the text in which we calculate the tf_idf for.
     returns a vector containing the tf_idf for all of the words in the list text
    """
    vector = []
    length_of_text = len(text) #the number of words in our text
    for i in range(length_of_text):
        word = text[i] #put the ith word of text into the variable word
        vector.append(tf(word, text) * idf(word, cleared_descriptions))
    return vector


def tf(word, text):
    """
     This method calculates the tf which is the number of times the word occurs in a text on the number of words in the text.
     word: the word we are calculating tf for
     text: the text in which we calculate the tf for word
    """
    number_of_occurances_in_text = 0 #number of times the word occurs in the text
    number_of_words_in_text = len(text) #total number of words in the text
    for i in range(number_of_words_in_text): #loop over the list text and count the number of occurances of the word
        if text[i] == word:
            number_of_occurances_in_text += 1
            
    return (number_of_occurances_in_text/number_of_words_in_text)


def idf(word, docs):
    """
     This method calculates the idf which is the logarithm (number of text in the corpus/number of texts where the word occurs).
     word: the word we are calculating idf for
     docs: all of our texts or documents
    """
    number_of_texts = len(docs) #total number of texts in our corpus
    number_of_texts_where_the_word_occurs = 0 #number of texts where the word occurs in the corpus
    for i in range(number_of_texts): #loop over all the documents or texts in the corpus and count the number of occurances of the word in corpus
        if word in docs[i]:
            number_of_texts_where_the_word_occurs += 1
    return (math.log((1 + number_of_texts)/(1 + number_of_texts_where_the_word_occurs)) + 1) #applying smoothing   


print(cleared_descriptions[2])

print(tf('اصل', cleared_descriptions[2]))

print(idf('اصل',cleared_descriptions))

non_numerical_title = cleared_titles # the list of processed words from the title to feed our model with
non_numerical_description = cleared_descriptions # the list of processed words from the descriptions to feed our model with
#Y is our actual values for cat1
non_numerical_Y = trainset['cat1'].head(1000) # we want to predict the cat1 with the title and description that we have for each text


def categorizer(category):
    """
    This method will get the list of words for categories and assigns some integer values to them instead. The integer values
    are as the following:
    businesses ==> 0
    electronic-devices ==> 1
    for-the-home ==> 2
    leisure-hobbies ==>3
    personal ==> 4
    vehicles ==> 5
    """
    classes = []
    category_length = len(category)
    for i in range(category_length):
        if category[i] == "businesses":
            classes.append(0)
            
        elif category[i] == "electronic-devices":
            classes.append(1)
            
        elif category[i] == "for-the-home":
            classes.append(2)
            
        elif category[i] == "leisure-hobbies":
            classes.append(3)
            
        elif category[i] == "personal":
            classes.append(4)
            
        else:
            classes.append(5)
            
    return classes
            


#in here we are going to create  a simple x for each text sample
def get_numerical_features(non_numerical_title, non_numerical_description):
    """
    This method returns the numerical features for the texts. One simple X for each text sample
    """
    numerical_title = []
    numerical_description = []
    for i in range(len(non_numerical_title)): #calculating a simple X for each text sample 
        numerical_title.append(np.mean(tf_idf(non_numerical_title[i])))
        numerical_description.append(np.mean(tf_idf(non_numerical_description[i])))
        
    return numerical_title, numerical_description


#A list containing numerical title and description for each text sample in the training set
titles, descriptions = get_numerical_features(non_numerical_title, non_numerical_description) 
Y = categorizer(non_numerical_Y) #A list containing an integer for the category in cat1

for i in range(5): #printing first 5 rows of X and their Corresponding Y
    print(titles[i]," ", descriptions[i]," " ,Y[i])

for i in range(5): #show some of the cat1 and their classes
    print(non_numerical_Y[i]," " ,Y[i])

# +
# Dividing the trainset into training set and test set. We need to create a test set out of our training
# set in order to update our model. In each itertion, we create a new test set out of our trainset and will try to predict them
# to see that is our model is good enough to go against the validationset.
# After the model is created, we go against the validation set with y = mx + b
# train_test_split, Splits arrays or matrices into random train and test subsets

# X_train : the training data out of our training set, used to train our model in each iteration

# X_test : the desired outputs for the X_train, used to train our model in each iteration 

# Y_train : part of the trainset that we seperate and use to predict Y_test to see if our model is good enough. If the model still
# is not good enough, then we update m and b of the linear function to predict better in the next iteration

# Y_test : the desired outputs for the Y_train. we compare these with the prediction of our model based on Y_train to calculate accuracy.
# if the accuracy is good enough, then the model is ready and we're done! 

#the following are all lists
title_train, title_test, description_train, description_test, Y_train, Y_test = train_test_split(
        titles, descriptions, Y, test_size=0.2, random_state=42)
# -

for i in range(5):
    print(title_train[i], title_test[i], description_train[i], description_test[i], Y_train[i], Y_test[i])

print(len(title_train)," ",len(title_test)," " ,len(description_train), " ",len(description_test), " ",len(Y_train), " ",len(Y_test))


def calculate_businesses(classes):
    """
    This method returns a list which contains one as an element, whenever the cat1 is business for that data and is zero otherwise
    """
    businesses = []
    classes_length = len(classes)
    for i in range(classes_length): #loop over the classes and if 0 which means the cat1 is businesses, append 1. append 0 otherwise
        if classes[i] == 0:
            businesses.append(1)
            
        else:
            businesses.append(0)
            
    return businesses



def calculate_electronic_devices(classes):
    """
    This method returns a list which contains one as an element, whenever the cat1 is electronic-devices for that 
    data and is zero otherwise
    """
    electronics = []
    classes_length = len(classes)
    for i in range(classes_length): #loop over the classes and if 0 which means the cat1 is businesses, append 1. append 0 otherwise
        if classes[i] == 1:
            electronics.append(1)
            
        else:
            electronics.append(0)
            
    return electronics



def calculate_for_the_home(classes):
    """
    This method returns a list which contains one as an element, whenever the cat1 is for-the-home 
    for that data and is zero otherwise
    """
    home = []
    classes_length = len(classes)
    for i in range(classes_length): #loop over the classes and if 0 which means the cat1 is businesses, append 1. append 0 otherwise
        if classes[i] == 2:
            home.append(1)
            
        else:
            home.append(0)
            
    return home



def calculate_leisure_hobbies(classes):
    """
    This method returns a list which contains one as an element, whenever the cat1 is leisure-hobbies 
    for that data and is zero otherwise
    """
    hobbies = []
    classes_length = len(classes)
    for i in range(classes_length): #loop over the classes and if 0 which means the cat1 is businesses, append 1. append 0 otherwise
        if classes[i] == 3:
            hobbies.append(1)
            
        else:
            hobbies.append(0)
            
    return hobbies



def calculate_personal(classes):
    """
    This method returns a list which contains one as an element, whenever the cat1 is personal 
    for that data and is zero otherwise
    """
    personal = []
    classes_length = len(classes)
    for i in range(classes_length): #loop over the classes and if 0 which means the cat1 is businesses, append 1. append 0 otherwise
        if classes[i] == 4:
            personal.append(1)
            
        else:
            personal.append(0)
            
    return personal



def calculate_vehicles(classes):
    """
    This method returns a list which contains one as an element, whenever the cat1 is vehicles 
    for that data and is zero otherwise
    """
    vehicles = []
    classes_length = len(classes)
    for i in range(classes_length): #loop over the classes and if 0 which means the cat1 is businesses, append 1. append 0 otherwise
        if classes[i] == 5:
            vehicles.append(1)
            
        else:
            vehicles.append(0)
            
    return vehicles



# These are the desired outputs for the entire trainset
y0 = calculate_businesses(Y_train) #y0 is a list which is one when the cat1 is businesses and zero otherwise
y1 = calculate_electronic_devices(Y_train) #y1 is a list which is one when the cat1 is electronic-devices and zero otherwise
y2 = calculate_for_the_home(Y_train) #y2 is a list which is one when the cat1 is for-the-home and zero otherwise
y3 = calculate_leisure_hobbies(Y_train) #y3 is a list which is one when the cat1 is leisure-hobbies and zero otherwise
y4 = calculate_personal(Y_train) #y4 is a list which is one when the cat1 is personal and zero otherwise
y5 = calculate_vehicles(Y_train) #y5 is a list which is one when the cat1 is vehicles and zero otherwise

for i in range(5): #printing first 5 rows of classes and their respective values in y0 to y5
    print(Y_train[i]," " , y0[i], y1[i], y2[i], y3[i], y4[i], y5[i])

# Thes are the desired outputs only for the train part of the trainset in the 6 class of categories
#We need these values in order to train our model to predict the X values in the validation set
Y_train0 = calculate_businesses(Y_train) #o0 is a list which is one when the cat1 in the Y_train is businesses and zero otherwise
Y_train1 = calculate_electronic_devices(Y_train) #o1 is a list which is one when the cat1 in the Y_train is electronic-devices and zero otherwise
Y_train2 = calculate_for_the_home(Y_train) #o2 is a list which is one when the cat1 in the Y_train is for-the-home and zero otherwise
Y_train3 = calculate_leisure_hobbies(Y_train) #o3 is a list which is one when the cat1 in the Y_train is leisure-hobbies and zero otherwise
Y_train4 = calculate_personal(Y_train) #o4 is a list which is one when the cat1 in the Y_train is personal and zero otherwise
Y_train5 = calculate_vehicles(Y_train) #o5 is a list which is one when the cat1 in the Y_train is vehicles and zero otherwise

for i in range(5):  #printing first 5 rows of classes_of_Y_train and their respective values in Y_train0 to Y_train5
    print(Y_train[i]," " , Y_train0[i], Y_train1[i], Y_train2[i], Y_train3[i], Y_train4[i], Y_train5[i])

# Implementing logistic regression: <br>
# Logistic Regression is a Machine Learning classification algorithm that is used to predict the probability of a categorical dependent variable.

# +
# Initializing logistic regression by determining the features and weights
feature = []
a = []
for i in range(len(title_train)):
    a.append(title_train[i])
    a.append(description_train[i])
    feature.append(a)    
    a = []

w0 = np.random.uniform(low = 0, high = 1) #initializing the wight0 with a random uniform real number
w1 = np.random.uniform(low = 0, high = 1) #initializing the wight1 with a random uniform real number
w2 = np.random.uniform(low = 0, high = 1) #initializing the wight2 with a random uniform real number

weights = np.array([w1, w2])
features = np.array(feature) #features is a 2d numpy array that has title_train as first column and description_train as the second 


# z = w0 + w1*title + w2*description

for i in range(5): #printing some features and their corresponding labels
    print(features[i])

print(features.shape)
print(weights.shape)
z = np.dot(features, weights)
print(len(z))
# -

# Initializing logistic regression for businesses
label = []
for i in range(len(title_train)):
    label.append(Y_train0[i]) #businesses
actual_business_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_business_labels.shape)

# Initializing logistic regression for electronic_devices
label = []
for i in range(len(title_train)):
    label.append(Y_train1[i]) #electronic_devices
actual_electronic_devices_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_electronic_devices_labels.shape)

# Initializing logistic regression for for_the_home
label = []
for i in range(len(title_train)):
    label.append(Y_train2[i]) #for_the_home
actual_for_the_home_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_for_the_home_labels.shape)

# Initializing logistic regression for leisure_hobbies
label = []
for i in range(len(title_train)):
    label.append(Y_train3[i]) #leisure_hobbies
actual_leisure_hobbies_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_leisure_hobbies_labels.shape)

# Initializing logistic regression for personal
label = []
for i in range(len(title_train)):
    label.append(Y_train4[i]) #personal
actual_personal_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_personal_labels.shape)

# Initializing logistic regression for vehicles
label = []
for i in range(len(title_train)):
    label.append(Y_train5[i]) #vehicles
actual_vehicles_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_vehicles_labels.shape)


def sigmoid(z):
    """
    Sigmoid function to map calculated value to a probablity
    """   
    return 1 / (1 + np.exp(-z)) #sigmoid(z) = 1/(1 + e^(-z))


def predict(features, weights):
  '''
  Returns 1D array of probabilities
  that the class label == 1
  '''
  z = np.dot(features, weights)
  return sigmoid(z)


# +
# print(predict(features, weights))
# print(np.log(predict(features, weights)))
# # print(np.log(1 - predict(features, weights)))
# print(labels)
# -

def cost_function(features, labels, weights):
    '''
    Using Mean Absolute Error

    Features:(400,2)
    Labels: (400,1)
    Weights:(2,1)
    Returns 1D matrix of predictions
    Cost = (labels*log(predictions) + (1-labels)*log(1-predictions) ) / len(labels)
    '''
    observations = len(labels)

    predictions = predict(features, weights)

    #Take the error when label=1, If label=0, the first side cancels out.
    class1_cost = -labels*np.log(predictions) 

    #Take the error when label=0, If label=1, the second side cancels out. 
    class2_cost = (1-labels)*np.log(1-predictions) #a negative number

    #Take the sum of both costs
    cost = class1_cost - class2_cost #a positive number

    #Take the average cost
    cost = cost.sum() / observations

    return cost


def update_weights(features, labels, weights, lr):
    '''
    Vectorized Gradient Descent

    Features:(200, 3)
    Labels: (200, 1)
    Weights:(3, 1)
    '''
    N = len(features)

    #1 - Get Predictions
    predictions = predict(features, weights)

    #2 Transpose features from (200, 3) to (3, 200)
    # So we can multiply w the (200,1)  cost matrix.
    # Returns a (3,1) matrix holding 3 partial derivatives --
    # one for each feature -- representing the aggregate
    # slope of the cost function across all observations
    gradient = np.dot(features.T,  predictions - labels)

    #3 Take the average cost derivative for each feature
    gradient /= N

    #4 - Multiply the gradient by our learning rate
    gradient *= lr

    #5 - Subtract from our weights to minimize cost
    weights -= gradient

    return weights


def decision_boundary(prob):
    return 1 if prob >= .5 else 0 


def classify(predictions):
    '''
    input  - N element array of predictions between 0 and 1
    output - N element array of 0s (False) and 1s (True)
    '''
    predicted_labels = []
    for i in range(len(predictions)):
        predicted_labels.append(decision_boundary(predictions[i]))
    return predicted_labels



def train(features, labels, weights, learning_rate, iters):
    cost_history = []

    for i in range(iters):
        weights = update_weights(features, labels, weights, learning_rate)

        #Calculate error for auditing purposes
        cost = cost_function(features, labels, weights)
        cost_history.append(cost)

        # Log Progress
        if i % 1000 == 0:
            print ("iter: ",str(i) , " cost: ",str(cost))

    return weights, cost_history


def accuracy(predicted_labels, actual_labels):
    diff = predicted_labels - actual_labels
    return 1.0 - (float(np.count_nonzero(diff)) / len(diff))


# +
# predictions = predict(features, weights)
# predicted_labels = classify(predictions) #labels predicted by the classifier
# print(predicted_labels)
# print(actual_business_labels)
# -

# TRAINING THE MODELS TO PREDICT EACH CATEGORY (y = w1 x title + w2 x description)

business_weights, business_cost_history = train(features, actual_business_labels, weights, 0.35, 10000)

business_predictions = predict(features, weights)
business_predicted_labels = classify(business_predictions) #labels predicted by the classifier

electronic_weights, electronic_cost_history = train(features, actual_electronic_devices_labels, weights, 0.35, 10000)

electronic_predictions = predict(features, weights)
electronic_predicted_labels = classify(electronic_predictions) #labels predicted by the classifier

for_the_home_weights, for_the_home_cost_history = train(features, actual_for_the_home_labels, weights, 0.35, 10000)

for_the_home_predictions = predict(features, weights)
for_the_home_predicted_labels = classify(for_the_home_predictions) #labels predicted by the classifier

hobbies_weights, hobbies_cost_history = train(features, actual_leisure_hobbies_labels, weights, 0.35, 10000)

hobbies_predictions = predict(features, weights)
hobbies_predicted_labels = classify(hobbies_predictions) #labels predicted by the classifier

personal_weights, personal_cost_history = train(features, actual_personal_labels, weights, 0.35, 10000)

personal_predictions = predict(features, weights)
personal_predicted_labels = classify(personal_predictions) #labels predicted by the classifier

vehicle_weights, vehicle_cost_history = train(features, actual_vehicles_labels, weights, 0.35, 10000)

vehicle_predictions = predict(features, weights)
vehicle_predicted_labels = classify(vehicle_predictions) #labels predicted by the classifier

# Accuracies For Each Model

print("Business Model:",accuracy(business_predicted_labels, actual_business_labels),"\n",
      "Electronic Model:", accuracy(electronic_predicted_labels, actual_electronic_devices_labels),"\n",
      "For-the-home Model:", accuracy(for_the_home_predicted_labels, actual_for_the_home_labels),"\n",
      "Hobbies Model:", accuracy(hobbies_predicted_labels, actual_leisure_hobbies_labels),"\n",
      "Personal Model:", accuracy(personal_predicted_labels, actual_personal_labels),"\n",
      "Vehicle Model:", accuracy(vehicle_predicted_labels, actual_vehicles_labels))


def softmax(x):
    """Using softmax to assign a class to each text sample"""
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum()


# Assigning values to each text sample using predictions acquired for each model and softmax function

print(business_predictions[0])
print(electronic_predictions[0])
print(for_the_home_predictions[0])
print(hobbies_predictions[0])
print(personal_predictions[0])
print(vehicle_predictions[0])
first_prediction = [business_predictions[0], electronic_predictions[0], for_the_home_predictions[0], 
              hobbies_predictions[0], personal_predictions[0], vehicle_predictions[0]]
probs = softmax(first_prediction).tolist()
print(probs)
print(probs.index(max(probs)))
# print(softmax(first_prediction).sum())
def multiclass_predictor(business_predictions, electronic_predictions, for_the_home_predictions, 
                 hobbies_predictions, personal_predictions, vehicle_predictions):
    """
    In this method, given the predictions done by each binary classifier, we assign a class to each text sample
    """  
    predicted_categories = []
    for i in range(len(business_predictions)):
        prediction = [business_predictions[i], electronic_predictions[i], for_the_home_predictions[i], 
              hobbies_predictions[i], personal_predictions[i], vehicle_predictions[i]] #list containing predictions
        probs = softmax(prediction).tolist() # a list of 6 values which each element rpresents the probability for that class
        predicted_categories.append(probs.index(max(probs))) #assigning the maximum probability as category
        
    return predicted_categories
        


final_predictions = multiclass_predictor(business_predictions, electronic_predictions, for_the_home_predictions, 
                 hobbies_predictions, personal_predictions, vehicle_predictions)

# Calculating the accuracy of the final model
final_predictions = np.array(final_predictions)
Y_train = np.array(Y_train)
final_model_accuracy = accuracy(final_predictions, Y_train)
print(final_model_accuracy)

for i in range(10):
    print(Y_train[i], final_predictions[i])


def calculate_cross_entropy_loss_function(predicted_y, actual_y):
    """
    cross entropy loss function calculates the distance between predicted output, and true output helping us to update our 
    weights and threshold, to have better predictions.
    predicted_y : the predicted output
    actual_y : the actual output
    """   
    return -(actual_y * math.log(predicted_y) + (1 - actual_y) * math.log(1 - predicted_y))


def gradient_descent(sigmoid, numerical_X_train, actual_y):
    """
    gradient descent algorithm, helps updating the wights.
    sigmoid : the predicted y, more precisely, the probability of being in the class
    numerical_X_train : the input data
    actual_y : the actual output
    """  
    derivatives = []
    for i in range(len(numerical_X_train)):
        corresponding_input_value = numerical_X_train[i]
        derivatives.append((sigmoid - actual_y) * corresponding_input_value)
    return derivatives


def update_weights(w, b, X, Y, learning_rate, previous_cost):
    """
    this method is used to update the weights and the threshold 
    w : the list of weights
    b : the threshold
    X : the input data, comes from a sample text
    learning_rate : the rate in which we wish to change our weights and threshold
    previous_cost : the cost before the updating
    """   
    latest_w = w
    latest_b = b
    w_deriv = 0 #derivative of the weights
    b_deriv = 0 #derivative of the threshold
    N = len(X) #the length of the input data
    for i in range(N):
        # Calculate partial derivatives
        # -2x(y - (wx + b))
        w_deriv += -2*X[i] * (Y - (w[i]*X[i] + b))
        # We subtract because the derivatives point in direction of steepest ascent
        w[i] -= (w_deriv / float(N)) * learning_rate
        
    # -2(y - (wx + b))
    b_deriv += -2*(Y - (np.mean(w)*X[0] + b))
    b -= (b_deriv / float(N)) * learning_rate
    z = weighted_sum_of_the_evidence_for_class(X, w, b) #calculating the weighted sum of the evidence for class
    probability_of_being_in_class = sigmoid(z) #calculating the probability of being in that specific class
    new_cost = calculate_cross_entropy_loss_function(probability_of_being_in_class, Y) #calculating the cost
    if(new_cost > previous_cost):
        w, b = latest_w, latest_b
    return w, b


# Calculating Scores

def calculate_true_positive(y_true, y_pred, category):
    """
    This method calculates the number of correct positive predictions for a specific class
    y_true : the list containing actual outputs
    y_pred : the list containing predicted outputs
    """  
    true_positives = 0
    length_of_data = len(y_true)
    #loop over the actual outputs and if the category in y_pred was also that specific class,
    #this means that the model correctly predicted positively
    for i in range(length_of_data): 
        if y_true[i] == category:
            if y_pred[i] == category:
                true_positives += 1 
    return true_positives


def calculate_true_negative(y_true, y_pred, category):
    """
    This method calculates the number of correct negative predictions for a specific class
    y_true : the list containing actual outputs
    y_pred : the list containing predicted outputs
    """ 
    true_negatives = 0
    length_of_data = len(y_true)
    #loop over the actual outputs and if the category was not that specific class, and also the y_pred was different
    #this means that the model correctly predicted negatively
    for i in range(length_of_data): 
        if y_true[i] != category:
            if y_pred[i] != category:
                true_negatives += 1 
    return true_negatives


def calculate_false_positive(y_true, y_pred, category):
    """
    This method calculates the number of false positive predictions for a specific class
    y_true : the list containing actual outputs
    y_pred : the list containing predicted outputs
    """ 
    false_positives = 0
    length_of_data = len(y_true)
    #loop over the predicted outputs; if the predicted output was category but the actual output (y_true) was not
    #this means that the model falsely predicted positively
    for i in range(length_of_data): 
        if y_pred[i] == category:
            if y_true[i] != category:
                false_positives += 1 
    return false_positives


def calculate_false_negative(y_true, y_pred, category):
    """
    This method calculates the number of false negative predictions for a specific class
    y_true : the list containing actual outputs
    y_pred : the list containing predicted outputs
    """ 
    false_negatives = 0
    length_of_data = len(y_true)
    #loop over the actual outputs and if the category was that specific class, and also the y_pred was different
    #this means that the model falsely predicted negatively
    for i in range(length_of_data): 
        if y_true[i] == category:
            if y_pred[i] != category:
                false_negatives += 1 
    return false_negatives


def calculate_accuracy(true_positive, true_negative, false_positive, false_negative):
    """
    This method calculates the accuracy of the model for a specific class
    true_positive : the samples that were positive and were predicted correctly
    true_negative : the samples that were negative and were predicted correctly
    false_positive : the samples that were negative and were falsely predicted as positive
    false_negative : the samples that were positive and were falsely predicted as negative
    """  
    accuracy = 0.0
    accuracy = round((true_positive + true_negative)/(true_positive + true_negative + false_positive + false_negative), 3)
    return accuracy


def calculate_percision(true_positive, false_positive):
    """
    This method calculates the percision of the model for a specific class
    true_positive : the samples that were positive and were predicted correctly
    false_positive : the samples that were negative and were falsely predicted as positive
    """  
    percision = 0.0
    percision = round(true_positive/(true_positive + false_positive) , 3)
    return percision


def calculate_recall(true_positive, false_negative):
    """
    This method calculates the recall of the model for a specific class
    true_positive : the samples that were positive and were predicted correctly
    false_negative : the samples that were positive and were falsely predicted as negative
    """  
    recall = 0.0
    recall = round(true_positive/(true_positive + false_negative) , 3)
    return recall


def calculate_f1_score(recall, percision):
    """
    This method calculates the f1 score of the model for a specific class
    percision : the percision of the model for that class
    recall : the recall of the model for that class
    """  
    f1_score = 0.0
    f1_score = round((2 * recall * percision)/(recall + percision) , 3)
    return f1_score


def calculate_support(y_true, category):
    """
    This method calculates the number of samples belong to a specific category in the y_true
    y_true : the list containing actual outputs
    category : the category that we are counting the number of
    """  
    count = 0
    length_of_data = len(y_true)
    for i in range(length_of_data):
        if y_true[i] == category:
            count += 1
    return count


def calculate_macro_average(f1_arr, percision_arr, recall_arr):
    """
    An arithmetic mean of per-class F1, percision, and recall
    f1_arr : a list containing f1_scores per class
    percision_arr : a list containing percision per class
    recall_arr : a list containing recall per class
    """ 
    macro_averaged_f1 = round(np.mean(f1_arr), 3)
    macro_averaged_percision = round(np.mean(percision_arr), 3)
    macro_averaged_recall = round(np.mean(recall_arr), 3)
    return macro_averaged_f1, macro_averaged_percision, macro_averaged_recall 


def calculate_weighted_average(f1_arr, percision_arr, recall_arr, y_true):
    """
    An arithmetic mean of per-class F1, percision, and recall, but also weights the score of each class by 
    the number of samplesfrom that class. This method works well based on the assumption that the f1_arr, 
    percision_arr, and recall_arr are given based on the alphabetic sort of their categories.
    f1_arr : a list containing f1_scores per class
    percision_arr : a list containing percision per class
    recall_arr : a list containing recall per class
    y_true : the list containing actual outputs
    """ 
    samples_length = len(y_true)
    counts = [] #a list containing the number of samples in each category
    myset = set(y_true) #converting to set
    categories = list(myset) #categories
    sorted_categories = sorted(categories)
#     print(sorted_categories)
    
    for i in range(len(categories)):
        counts.append(y_true.count(sorted_categories[i]))
#     print(counts)
    f1 = 0
    prec = 0
    rec = 0
#     for i in range(len(f1_arr)):
#         print(f1_arr[i], " ", counts[i])
    for i in range(len(f1_arr)):
        f1 += (f1_arr[i] * counts[i])  
        prec += (percision_arr[i] * counts[i])
        rec += (recall_arr[i] * counts[i])
    weighted_f1 = round(f1/samples_length, 3)  
    weighted_percision = round(prec/samples_length, 3)
    weighted_recall = round(rec/samples_length, 3)
    
    return weighted_f1, weighted_percision, weighted_recall 


# +
def calculate_micro_average(confusion_matrix):
    """
    The following always holds true for the micro-F1 case:
    micro-F1 = micro_percision = micro_recall = accuracy
    We first calculate micro_percision and micro_recall and then combine the two
    f1_arr : a list containing f1_scores per class
    percision_arr : a list containing percision per class
    recall_arr : a list containing recall per class
    y_true : the list containing actual outputs
    """ 
    #in multi-class, all the correctly predicted samples are true positives.
    #gettin the diagonal
    true_positives = 0
    j = 0
    for i in range(len(confusion_matrix)):
        true_positives += confusion_matrix[i][j]
        j += 1
    
#     print(true_positives)
    #each prediction error is a false positive for the class that we predicted
    #again, the total number of false negatives is the total number of prediction errors
    #false_negatives = false_positives
    false_positives = 0
    for i in range(len(confusion_matrix)): #looping over the confusion matrix
        for j in range(len(confusion_matrix)):
            if i != j:  #passing the diagonal
                false_positives += confusion_matrix[i][j]
#     print(false_positives)    
    accuracy = true_positives/(true_positives + false_positives)  
    return round(accuracy, 3)
        


# +
from sklearn import metrics
# Constants
B = "businesses"
E = "electronic-devices"
F = "for-the-home"
L = "leisure-hobbies"
P = "personal"
V = "vehicles"

# True values
y_true = [B, E, B, E, F, L, V, V, V, P, V, B, E, F, L, P, V]
# Predicted values
y_pred = [E, B, F, L, L, L, V, P, P, B, V, B, E, F, L, P, V]

# Print the confusion matrix
print(metrics.confusion_matrix(y_true, y_pred))

# Print the precision and recall, among other metrics
print(metrics.classification_report(y_true, y_pred, digits=3))

# +
from sklearn import metrics
# Constants
C="Cat"
F="Fish"
H="Hen"

# True values
y_true = [C,C,C,C,C,C, F,F,F,F,F,F,F,F,F,F, H,H,H,H,H,H,H,H,H]
# Predicted values
y_pred = [C,C,C,C,H,F, C,C,C,C,C,C,H,H,F,F, C,C,C,H,H,H,H,H,H]

# Print the confusion matrix
confusion_matrix = metrics.confusion_matrix(y_true, y_pred)
print(confusion_matrix)

# Print the precision and recall, among other metrics
print(metrics.classification_report(y_true, y_pred, digits=3))
# -

#based on the category of the previous four calculations for TP, TN, FP, FN, 
# the acuracy, percision, recall and f1 score will be calculated for the same category
true_positive = calculate_true_positive(y_true, y_pred, C) 
true_negative = calculate_true_negative(y_true, y_pred, C)
false_positive = calculate_false_positive(y_true, y_pred, C)
false_negative = calculate_false_negative(y_true, y_pred, C)
accuracy_for_cat = calculate_accuracy(true_positive, true_negative, false_positive, false_negative)
percision_for_cat = calculate_percision(true_positive, false_positive)
recall_for_cat = calculate_recall(true_positive, false_negative)
f1_score_for_cat = calculate_f1_score(recall_for_cat, percision_for_cat)

print(true_positive, " ", true_negative, " ", false_positive, " ", false_negative) 
print(accuracy_for_cat, " ", percision_for_cat, " ", recall_for_cat, " ", f1_score_for_cat)

f1 = [0.421, 0.308, 0.667]
percision = [0.308, 0.667, 0.667]
recall = [0.667, 0.200, 0.667]
macro_f1, macro_percision, macro_recall = calculate_macro_average(f1, percision, recall)
weighted_f1, weighted_percision, weighted_recall = calculate_weighted_average(f1, percision, recall, y_true)
accuracy =  calculate_micro_average(confusion_matrix)

f1 = [0.333, 0.400, 0.500, 0.667, 0.400, 0.750]
percision = [0.333, 0.500, 0.500, 0.500, 0.333, 1.000]
recall = [0.333, 0.333, 0.500, 1.000, 0.500, 0.600]
macro_f1, macro_percision, macro_recall = calculate_macro_average(f1, percision, recall)
weighted_f1, weighted_percision, weighted_recall = calculate_weighted_average(f1, percision, recall, y_true)
accuracy =  calculate_micro_average(confusion_matrix)

print(macro_f1, " ", macro_percision, " ", macro_recall) 

print(weighted_f1, " ", weighted_percision, " ", weighted_recall)

print(accuracy)

# the average methods need fixing to do the job for other dataframes.
# the confusion matrix function should be implemented

# +
from sklearn import metrics


# Print the confusion matrix
print(metrics.confusion_matrix(Y_train, final_predictions))

# Print the precision and recall, among other metrics
print(metrics.classification_report(Y_train, final_predictions, digits=3))
# -

# implementation of confusion matrix
import pandas as pd
y_actu = pd.Series(Y_train, name='Actual')
y_pred = pd.Series(final_predictions, name='Predicted')
df_confusion = pd.crosstab(y_actu, y_pred, rownames=['Actual'], colnames=['Predicted'], margins=True)

df_confusion

# calcularing the accuracy for validation set
descriptions = validationset['desc']
titles = validationset['title']
cleared_descriptions = preprocessing(descriptions.head(1000)) #preprocessed descriptions
cleared_titles = preprocessing(titles.head(1000)) #preprocessed titles
#the following are all lists
#A list containing numerical title and description for each text sample in the training set
titles, descriptions = get_numerical_features(non_numerical_title, non_numerical_description) 
Y = categorizer(non_numerical_Y) #A list containing an integer for the category in cat1
title_train, title_test, description_train, description_test, Y_train, Y_test = train_test_split(
        titles, descriptions, Y, test_size=0.2, random_state=42)

# Thes are the desired outputs only for the train part of the trainset in the 6 class of categories
#We need these values in order to train our model to predict the X values in the validation set
Y_train0 = calculate_businesses(Y_train) #o0 is a list which is one when the cat1 in the Y_train is businesses and zero otherwise
Y_train1 = calculate_electronic_devices(Y_train) #o1 is a list which is one when the cat1 in the Y_train is electronic-devices and zero otherwise
Y_train2 = calculate_for_the_home(Y_train) #o2 is a list which is one when the cat1 in the Y_train is for-the-home and zero otherwise
Y_train3 = calculate_leisure_hobbies(Y_train) #o3 is a list which is one when the cat1 in the Y_train is leisure-hobbies and zero otherwise
Y_train4 = calculate_personal(Y_train) #o4 is a list which is one when the cat1 in the Y_train is personal and zero otherwise
Y_train5 = calculate_vehicles(Y_train) #o5 is a list which is one when the cat1 in the Y_train is vehicles and zero otherwise

# +
# Initializing logistic regression by determining the features and weights
feature = []
a = []
for i in range(len(title_train)):
    a.append(title_train[i])
    a.append(description_train[i])
    feature.append(a)    
    a = []

w0 = np.random.uniform(low = 0, high = 1) #initializing the wight0 with a random uniform real number
w1 = np.random.uniform(low = 0, high = 1) #initializing the wight1 with a random uniform real number
w2 = np.random.uniform(low = 0, high = 1) #initializing the wight2 with a random uniform real number

weights = np.array([w1, w2])
features = np.array(feature) #features is a 2d numpy array that has title_train as first column and description_train as the second 


# z = w0 + w1*title + w2*description

for i in range(5): #printing some features and their corresponding labels
    print(features[i])

print(features.shape)
print(weights.shape)
z = np.dot(features, weights)
print(len(z))
# -

# TRAINING THE MODELS TO PREDICT EACH CATEGORY (y = w1 x title + w2 x description)

# +
# Initializing logistic regression for businesses
label = []
for i in range(len(title_train)):
    label.append(Y_train0[i]) #businesses
actual_business_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_business_labels.shape)

# Initializing logistic regression for electronic_devices
label = []
for i in range(len(title_train)):
    label.append(Y_train1[i]) #electronic_devices
actual_electronic_devices_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_electronic_devices_labels.shape)

# Initializing logistic regression for for_the_home
label = []
for i in range(len(title_train)):
    label.append(Y_train2[i]) #for_the_home
actual_for_the_home_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_for_the_home_labels.shape)

# Initializing logistic regression for leisure_hobbies
label = []
for i in range(len(title_train)):
    label.append(Y_train3[i]) #leisure_hobbies
actual_leisure_hobbies_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_leisure_hobbies_labels.shape)

# Initializing logistic regression for personal
label = []
for i in range(len(title_train)):
    label.append(Y_train4[i]) #personal
actual_personal_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_personal_labels.shape)

# Initializing logistic regression for vehicles
label = []
for i in range(len(title_train)):
    label.append(Y_train5[i]) #vehicles
actual_vehicles_labels = np.array(label) #labels are the desires output or Y_train also an numpy array
print(actual_vehicles_labels.shape)

# -

business_weights, business_cost_history = train(features, actual_business_labels, weights, 0.35, 10000)
business_predictions = predict(features, weights)
business_predicted_labels = classify(business_predictions) #labels predicted by the classifier

electronic_weights, electronic_cost_history = train(features, actual_electronic_devices_labels, weights, 0.35, 10000)
electronic_predictions = predict(features, weights)
electronic_predicted_labels = classify(electronic_predictions) #labels predicted by the classifier

for_the_home_weights, for_the_home_cost_history = train(features, actual_for_the_home_labels, weights, 0.35, 10000)
for_the_home_predictions = predict(features, weights)
for_the_home_predicted_labels = classify(for_the_home_predictions) #labels predicted by the classifier

hobbies_weights, hobbies_cost_history = train(features, actual_leisure_hobbies_labels, weights, 0.35, 10000)
hobbies_predictions = predict(features, weights)
hobbies_predicted_labels = classify(hobbies_predictions) #labels predicted by the classifier

personal_weights, personal_cost_history = train(features, actual_personal_labels, weights, 0.35, 10000)
personal_predictions = predict(features, weights)
personal_predicted_labels = classify(personal_predictions) #labels predicted by the classifier

vehicle_weights, vehicle_cost_history = train(features, actual_vehicles_labels, weights, 0.35, 10000)
vehicle_predictions = predict(features, weights)
vehicle_predicted_labels = classify(vehicle_predictions) #labels predicted by the classifier

print("Business Model:",accuracy(business_predicted_labels, actual_business_labels),"\n",
      "Electronic Model:", accuracy(electronic_predicted_labels, actual_electronic_devices_labels),"\n",
      "For-the-home Model:", accuracy(for_the_home_predicted_labels, actual_for_the_home_labels),"\n",
      "Hobbies Model:", accuracy(hobbies_predicted_labels, actual_leisure_hobbies_labels),"\n",
      "Personal Model:", accuracy(personal_predicted_labels, actual_personal_labels),"\n",
      "Vehicle Model:", accuracy(vehicle_predicted_labels, actual_vehicles_labels))

# +
# multiclass prediction
final_predictions = multiclass_predictor(business_predictions, electronic_predictions, for_the_home_predictions, 
                 hobbies_predictions, personal_predictions, vehicle_predictions)

# Calculating the accuracy of the final model
final_predictions = np.array(final_predictions)
Y_train = np.array(Y_train)
final_model_accuracy = accuracy(final_predictions, Y_train)
print(final_model_accuracy)
for i in range(10):
    print(Y_train[i], final_predictions[i])
# -



StarcoderdataJupyterScriptsDedupFiltered