<jupyter_start><jupyter_text>XLNet training for Personality Detection Neuroticism* Based on: https://medium.com/swlh/using-xlnet-for-sentiment-classification-cfa948e65e85  Configuración de Bibliotecas<jupyter_code>import transformers
from transformers import XLNetTokenizer, XLNetModel, AdamW, get_linear_schedule_with_warmup
import torch

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib import rc
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
from collections import defaultdict
from textwrap import wrap
from pylab import rcParams

from torch import nn, optim
from keras.preprocessing.sequence import pad_sequences
from torch.utils.data import TensorDataset,RandomSampler,SequentialSampler
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F

import re

import json
import random<jupyter_output><empty_output><jupyter_text>Obtener los datos<jupyter_code>with open("../jsons/twitter/guide.json", "r") as guidejson:
    json_dic = json.load(guidejson)
    
not_neu_count = json_dic["0"]
neu_count = json_dic["1"]
zero_neu = []
one_neu = []

for i in range(1,not_neu_count + 1):
    with open(f'../jsons/twitter/{i}-0.json', "r") as temp_not_neu:
        temp_json = json.load(temp_not_neu)
        temp_json["y"] = 0
        zero_neu.append(temp_json)

for i in range(1,neu_count + 1):
    with open(f'../jsons/twitter/{i}-1.json', "r") as temp_neu:
        temp_json = json.load(temp_neu)
        temp_json["y"] = 1
        one_neu.append(temp_json)
    
print(len(zero_neu), len(one_neu))<jupyter_output>6200 3717<jupyter_text>Preparar los datos para la XLNet<jupyter_code>import random

random.seed(1)

random.shuffle(zero_neu)
random.shuffle(one_neu)

train_data = []
test_data = []

train_seg = 0.7

for i in zero_neu[:int(len(zero_neu) * train_seg)]:
    train_data.append(i)
    
for i in one_neu[:int(len(one_neu) * train_seg)]:
    train_data.append(i)
    
for i in zero_neu[int(len(zero_neu) * train_seg):]:
    test_data.append(i)

for i in one_neu[int(len(one_neu) * train_seg):]:
    test_data.append(i)
    
random.shuffle(train_data)
random.shuffle(test_data)<jupyter_output><empty_output><jupyter_text>Load Model<jupyter_code>from transformers import XLNetForSequenceClassification

model = XLNetForSequenceClassification.from_pretrained('xlnet-base-cased', num_labels = 2)<jupyter_output>Some weights of the model checkpoint at xlnet-base-cased were not used when initializing XLNetForSequenceClassification: ['lm_loss.weight', 'lm_loss.bias']
- This IS expected if you are initializing XLNetForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing XLNetForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of XLNetForSequenceClassification were not initialized from the model checkpoint at xlnet-base-cased and are newly initialized: ['sequence_summary.summary.weight', 'sequence_summary.summary.bias', 'logits_proj.weight', 'logits_proj.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions a[...]<jupyter_text>Required functions<jupyter_code>def data_segmenter(elem):
    input_ids = []
    attention_mask = []

    input_ids = elem['input_ids']
    attention_mask = elem['attention_mask']
    y = torch.Tensor([elem["y"]]).long()
    
    input_ids = torch.Tensor(input_ids).long()
    attention_mask = torch.Tensor(attention_mask).long()
    
    return input_ids, attention_mask, y

ids, am, y = data_segmenter(train_data[0])
print(ids.shape)
from sklearn import metrics
def train_twitter(input_ids, attention_mask, targets, optimizer, scheduler):

    acc = 0

    outputs = model(input_ids=input_ids, token_type_ids=None, 
                    attention_mask=attention_mask, labels = targets)
    loss = outputs[0]
    logits = outputs[1]
    # preds = preds.cpu().detach().numpy()
    _, prediction = torch.max(outputs[1], dim=1)
#     print(prediction)
    targets = targets.cpu().detach().numpy()
    prediction = prediction.cpu().detach().numpy()
    accuracy = metrics.accuracy_score(targets, prediction)

    acc = accuracy

    loss.backward()

    nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
    optimizer.step()
    scheduler.step()
    optimizer.zero_grad()

    return acc, loss.item()
def eval_model(input_ids, attention_mask, targets):
    
    loss = 0
    acc = 0
  
    with torch.no_grad():
        outputs = model(input_ids=input_ids, token_type_ids=None,
                        attention_mask=attention_mask, labels = targets)
        loss = outputs[0]
        logits = outputs[1]

        _, prediction = torch.max(outputs[1], dim=1)
        targets = targets.cpu().detach().numpy()
        prediction = prediction.cpu().detach().numpy()
        accuracy = metrics.accuracy_score(targets, prediction)

        acc += accuracy

    return acc, loss.item()<jupyter_output><empty_output><jupyter_text>Global Variables<jupyter_code>EPOCHS = 10
# BATCH_SIZE = 2
# DATA_SIZE = 512

param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
                                {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
                                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay':0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5)

total_steps = len(train_data) * EPOCHS

scheduler = get_linear_schedule_with_warmup(
  optimizer,
  num_warmup_steps=0,
  num_training_steps=total_steps
)
import time<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>history_train = defaultdict(list)
best_accuracy = 0


start_time = time.time()


for epoch in range(EPOCHS):
    train_acc = 0
    train_loss = []
    it = 0
    model = model.train()
    
    while it < len(train_data):
        
        
        input_ids, attention_mask, targets = data_segmenter(train_data[it])
        
        temp_acc, temp_loss = train_twitter(
            input_ids, 
            attention_mask, 
            targets, 
            optimizer, 
            scheduler
        )
    
        train_acc += temp_acc
        train_loss.append(temp_loss)
        
        it += 1
        
        print(f"[EPOCH {epoch + 1}, BATCH {it} of {len(train_data)}] {targets[0]} ->ACC: {train_acc/it}")
        with open(f'Results/ACC/acum_acc_train_epoch_{epoch+1}.txt', 'w') as acc_file:
            acc_file.write(f"[EPOCH {epoch + 1}, BATCH {it} of {len(train_data)}] ->ACC: {train_acc/it}")
        
    history_train['train_acc'].append(train_acc / it)
    history_train['train_loss'].append(np.mean(train_loss))
    
    with open('history_train.json', 'w') as hist_json:
        json.dump(history_train, hist_json)
    
    model.save_pretrained(f'Results/Models/Model-{epoch+1}')
    
    if best_accuracy < history_train["train_acc"][-1]:
        best_accuracy = history_train["train_acc"][-1]
    
    print(f'Train loss {history_train["train_loss"][-1]} Train accuracy {history_train["train_acc"][-1]}')
    print()

print("Losses:")
print(history_train['train_loss'])
print("Accuracy:")
print(history_train['train_acc'])

print("Best Accuracy:", best_accuracy)

total_time = time.time() - start_time

with open('train_time.txt', 'w') as time_file:
    time_file.write(f'Total_time: {total_time}')<jupyter_output>[EPOCH 1, BATCH 1 of 6941] 1 ->ACC: 1.0
[EPOCH 1, BATCH 2 of 6941] 0 ->ACC: 0.5
[EPOCH 1, BATCH 3 of 6941] 0 ->ACC: 0.3333333333333333
[EPOCH 1, BATCH 4 of 6941] 0 ->ACC: 0.5
[EPOCH 1, BATCH 5 of 6941] 0 ->ACC: 0.4
[EPOCH 1, BATCH 6 of 6941] 0 ->ACC: 0.5
[EPOCH 1, BATCH 7 of 6941] 0 ->ACC: 0.5714285714285714
[EPOCH 1, BATCH 8 of 6941] 0 ->ACC: 0.625
[EPOCH 1, BATCH 9 of 6941] 0 ->ACC: 0.6666666666666666
[EPOCH 1, BATCH 10 of 6941] 1 ->ACC: 0.6
[EPOCH 1, BATCH 11 of 6941] 0 ->ACC: 0.6363636363636364
[EPOCH 1, BATCH 12 of 6941] 1 ->ACC: 0.5833333333333334
[EPOCH 1, BATCH 13 of 6941] 1 ->ACC: 0.5384615384615384
[EPOCH 1, BATCH 14 of 6941] 0 ->ACC: 0.5714285714285714
[EPOCH 1, BATCH 15 of 6941] 0 ->ACC: 0.6
[EPOCH 1, BATCH 16 of 6941] 1 ->ACC: 0.5625
[EPOCH 1, BATCH 17 of 6941] 0 ->ACC: 0.5882352941176471
[EPOCH 1, BATCH 18 of 6941] 0 ->ACC: 0.6111111111111112
[EPOCH 1, BATCH 19 of 6941] 0 ->ACC: 0.631578947368421
[EPOCH 1, BATCH 20 of 6941] 0 ->ACC: 0.65
[EPOCH 1, BATCH 21 of 6941] 0 ->AC[...]<jupyter_text>Test<jupyter_code>history_test = defaultdict(list)
best_accuracy = 0

train_acc = 0
train_loss = []
it = 0

start_time = time.time()



for epoch in range(EPOCHS):
    test_acc = 0
    test_loss = []
    it = 0

    model = XLNetForSequenceClassification.from_pretrained(f'Model-{epoch+1}', num_labels = 2)
    model = model.eval()
    
    while it < len(test_data):
            
        input_ids, attention_mask, targets = data_segmenter(test_data[it])
        
        temp_acc, temp_loss = eval_model(
            model,
            input_ids, 
            attention_mask, 
            targets
        )

        test_acc += temp_acc
        test_loss.append(temp_loss)

        it += 1

        print(f"[EPOCH {epoch+1}, BATCH {it} of {len(test_data)}] {targets[0]} ->AC: {test_acc/it}")

        with open(f'Results/ACC/acum_acc_test_epoch_{epoch+1}.txt', 'w') as acc_file:
            acc_file.write(f"[EPOCH {epoch + 1}, BATCH {it} of {len(test_data)}] ->ACC: {train_acc/it}")
        

    history_test['test_acc'].append(test_acc / it)
    history_test['test_loss'].append(np.mean(test_loss))

    print(f'Test loss {history_test["test_loss"][-1]} Test accuracy {history_test["test_acc"][-1]}')
    print()
    
    if best_accuracy < history_test["test_acc"][-1]:
        best_accuracy = history_test["test_acc"][-1]

    with open('history_test.json', 'w') as hist_json:
        json.dump(history_test, hist_json)
        

print("Losses:")
print(history_test['test_loss'])
print("Accuracy:")
print(history_test['test_acc'])

print("Best Accuracy:", best_accuracy)

total_time = time.time() - start_time

with open('test_time.txt', 'w') as time_file:
    time_file.write(f'Total_time: {total_time}')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup