# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + colab={"base_uri": "https://localhost:8080/"} id="yVzyeZQ-xY09" outputId="da308cb3-8cf2-49a1-a0fd-ff994320cca2"
# !pip install transformers
# !pip install sentencepiece

# + id="1STnA6oVy0UW"
import os
import time
import datetime
import random

import numpy as np
import pandas as pd
import tensorflow as tf
import torch

from transformers import XLMRobertaForSequenceClassification, XLMRobertaTokenizer, AdamW, AutoTokenizer, get_linear_schedule_with_warmup
from torch.utils.data import TensorDataset, random_split, DataLoader, RandomSampler, SequentialSampler

# + colab={"base_uri": "https://localhost:8080/"} id="aAOAvWUUvyAV" outputId="b673aec8-53f3-4cf1-c2cc-fa4f085d3c0a"
# Get the GPU device name.
device_name = tf.test.gpu_device_name()

# The device name should look like the following:
if device_name == '/device:GPU:0':
    print('Found GPU at: {}'.format(device_name))
else:
    raise SystemError('GPU device not found')

# + colab={"base_uri": "https://localhost:8080/"} id="Bvxjt6Vu-sh-" outputId="c7f19a1c-53d0-49d7-81aa-4b9451eeb4e2"

# If there's a GPU available...
if torch.cuda.is_available():    

    # Tell PyTorch to use the GPU.    
    device = torch.device("cuda")

    print('There are %d GPU(s) available.' % torch.cuda.device_count())

    print('We will use the GPU:', torch.cuda.get_device_name(0))

# If not...
else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")

# + [markdown] id="bREyiAum7Cyi"
# #### Format data

# + id="j2pfChozyLhm"
base_path = "/content/drive/MyDrive/Colab-data/"

en_train_tweets_file = "english_train_tweets.csv"
# en_test_tweets_file = "english_test_tweets.csv"
ar_tweets_file = "arabic_test_tweets.csv"

# Load the dataset into a pandas dataframe.
en_tweets_df = pd.read_csv(base_path + en_train_tweets_file)
ar_tweets_df = pd.read_csv(base_path + ar_tweets_file)

en_tweets_df['sentiment'].replace({'positive': 1, 'negative': 2, 'neutral': 0}, inplace=True)
ar_tweets_df['sentiment'].replace({'positive': 1, 'negative': 2, 'neutral': 0}, inplace=True)

# + colab={"base_uri": "https://localhost:8080/"} id="X0lTaEDdue68" outputId="870757eb-a82f-4f3f-b9a4-17956e3318ba"
total_arabic_tweets = ar_tweets_df.shape[0]
train_size = int(0.9 * total_arabic_tweets)

ar_train_df = ar_tweets_df[:train_size]
ar_test_df = ar_tweets_df[train_size:]

print(f"total english tweets {en_tweets_df.shape[0]}")
print(f"total arabic tweets {total_arabic_tweets}")
print(f"train arabic tweets {ar_train_df.shape[0]}")
print(f"test arabic tweets {ar_test_df.shape[0]}")

# + colab={"base_uri": "https://localhost:8080/", "height": 529} id="eDjf5QWixyBy" outputId="c5cc389e-bf79-422e-c82f-b51667e5b31c"
en_tweets_df.sample(10)

# + colab={"base_uri": "https://localhost:8080/", "height": 410} id="yjI6FsqL07AD" outputId="7f6987d4-61a3-43a9-f2f6-af8eafe13cc2"
ar_train_df.sample(10)

# + colab={"base_uri": "https://localhost:8080/", "height": 495} id="dGFqy1vL09q0" outputId="4db858f8-9e86-49d2-9703-68c1e3bb216f"
ar_test_df.sample(10)


# + [markdown] id="TbRgtVxX69zY"
# #### Define Helper functions

# + id="yP_e2mT944Hc"
def tokenize_sentences(sentences, labels, tokenizer):
  # Tokenize all of the sentences and map the tokens to thier word IDs.
  input_ids = []
  attention_masks = []

  # For every sentence...
  for sent in sentences:
      # `encode_plus` will:
      #   (1) Tokenize the sentence.
      #   (2) Prepend the `[CLS]` token to the start.
      #   (3) Append the `[SEP]` token to the end.
      #   (4) Map tokens to their IDs.
      #   (5) Pad or truncate the sentence to `max_length`
      #   (6) Create attention masks for [PAD] tokens.
      encoded_dict = tokenizer.encode_plus(
                          sent,                      # Sentence to encode.
                          add_special_tokens = True, # Add '[CLS]' and '[SEP]'
                          max_length = 64,           # Pad & truncate all sentences.
                          pad_to_max_length = True,
                          return_attention_mask = True,   # Construct attn. masks.
                          return_tensors = 'pt',     # Return pytorch tensors.
                    )
      
      # Add the encoded sentence to the list.    
      input_ids.append(encoded_dict['input_ids'])
      
      # And its attention mask (simply differentiates padding from non-padding).
      attention_masks.append(encoded_dict['attention_mask'])

  # Convert the lists into tensors.
  input_ids = torch.cat(input_ids, dim=0)
  attention_masks = torch.cat(attention_masks, dim=0)
  labels = torch.tensor(labels)

  # Print sentence 0, now as a list of IDs.
  print('Original: ', sentences[0])
  print('Token IDs:', input_ids[0])

  return input_ids, attention_masks, labels


# + id="CyMp-ioT9SfL"
def train_val_split(input_ids, attention_masks, labels):
  # Combine the training inputs into a TensorDataset.
  dataset = TensorDataset(input_ids, attention_masks, labels)

  # Create a 90-10 train-validation split.

  # Calculate the number of samples to include in each set.
  train_size = int(0.9 * len(dataset))
  val_size = len(dataset) - train_size

  # Divide the dataset by randomly selecting samples.
  train_dataset, val_dataset = random_split(dataset, [train_size, val_size])

  return train_dataset, val_dataset, train_size, val_size


# + id="T7LyXh-h9z-Y"
def get_data_loaders(train_dataset, val_dataset, batch_size = 32):
  # The DataLoader needs to know our batch size for training, so we specify it 
  # here. For fine-tuning BERT on a specific task, the authors recommend a batch 
  # size of 16 or 32.
  # batch_size = 32

  # Create the DataLoaders for our training and validation sets.
  # We'll take training samples in random order. 
  train_dataloader = DataLoader(
              train_dataset,  # The training samples.
              sampler = RandomSampler(train_dataset), # Select batches randomly
              batch_size = batch_size # Trains with this batch size.
          )

  # For validation the order doesn't matter, so we'll just read them sequentially.
  validation_dataloader = DataLoader(
              val_dataset, # The validation samples.
              sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.
              batch_size = batch_size # Evaluate with this batch size.
          )
  return train_dataloader, validation_dataloader


# + id="HTfgzob2-J-y"
epochs = 4

def get_lr_scheduler(train_dataloader):

  # Total number of training steps is [number of batches] x [number of epochs]. 
  # (Note that this is not the same as the number of training samples).
  total_steps = len(train_dataloader) * epochs

  # Create the learning rate scheduler.
  scheduler = get_linear_schedule_with_warmup(optimizer, 
                                              num_warmup_steps = 0, # Default value in run_glue.py
                                              num_training_steps = total_steps)
  return scheduler


# + id="DqvLIHi3-U2L"
# Function to calculate the accuracy of our predictions vs labels
def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)


# + id="PmnHIFZE-bVR"
def format_time(elapsed):
    '''
    Takes a time in seconds and returns a string hh:mm:ss
    '''
    # Round to the nearest second.
    elapsed_rounded = int(round((elapsed)))
    
    # Format as hh:mm:ss
    return str(datetime.timedelta(seconds=elapsed_rounded))


# + id="Q8fhmMw9-jRt"
def train(model, epochs, train_dataloader, validation_dataloader, scheduler, seed_val=42):
  # This training code is based on the `run_glue.py` script here:
  # https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128

  # Set the seed value all over the place to make this reproducible.
  # seed_val = 42

  random.seed(seed_val)
  np.random.seed(seed_val)
  torch.manual_seed(seed_val)
  torch.cuda.manual_seed_all(seed_val)

  # We'll store a number of quantities such as training and validation loss, 
  # validation accuracy, and timings.
  training_stats = []

  # Measure the total training time for the whole run.
  total_t0 = time.time()

  # For each epoch...
  for epoch_i in range(0, epochs):
      
      # ========================================
      #               Training
      # ========================================
      
      # Perform one full pass over the training set.

      print("")
      print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
      print('Training...')

      # Measure how long the training epoch takes.
      t0 = time.time()

      # Reset the total loss for this epoch.
      total_train_loss = 0

      # Put the model into training mode. Don't be mislead--the call to 
      # `train` just changes the *mode*, it doesn't *perform* the training.
      # `dropout` and `batchnorm` layers behave differently during training
      # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)
      model.train()

      # For each batch of training data...
      for step, batch in enumerate(train_dataloader):

          # Progress update every 80 batches.
          if step % 80 == 0 and not step == 0:
              # Calculate elapsed time in minutes.
              elapsed = format_time(time.time() - t0)
              
              # Report progress.
              print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))

          # Unpack this training batch from our dataloader. 
          #
          # As we unpack the batch, we'll also copy each tensor to the GPU using the 
          # `to` method.
          #
          # `batch` contains three pytorch tensors:
          #   [0]: input ids 
          #   [1]: attention masks
          #   [2]: labels 
          b_input_ids = batch[0].to(device)
          b_input_mask = batch[1].to(device)
          b_labels = batch[2].to(device)

          # Always clear any previously calculated gradients before performing a
          # backward pass. PyTorch doesn't do this automatically because 
          # accumulating the gradients is "convenient while training RNNs". 
          # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)
          model.zero_grad()        

          # Perform a forward pass (evaluate the model on this training batch).
          # In PyTorch, calling `model` will in turn call the model's `forward` 
          # function and pass down the arguments. The `forward` function is 
          # documented here: 
          # https://huggingface.co/transformers/model_doc/bert.html#bertforsequenceclassification
          # The results are returned in a results object, documented here:
          # https://huggingface.co/transformers/main_classes/output.html#transformers.modeling_outputs.SequenceClassifierOutput
          # Specifically, we'll get the loss (because we provided labels) and the
          # "logits"--the model outputs prior to activation.
          result = model(b_input_ids, 
                        token_type_ids=None, 
                        attention_mask=b_input_mask, 
                        labels=b_labels,
                        return_dict=True)

          loss = result.loss
          logits = result.logits

          # Accumulate the training loss over all of the batches so that we can
          # calculate the average loss at the end. `loss` is a Tensor containing a
          # single value; the `.item()` function just returns the Python value 
          # from the tensor.
          total_train_loss += loss.item()

          # Perform a backward pass to calculate the gradients.
          loss.backward()

          # Clip the norm of the gradients to 1.0.
          # This is to help prevent the "exploding gradients" problem.
          torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

          # Update parameters and take a step using the computed gradient.
          # The optimizer dictates the "update rule"--how the parameters are
          # modified based on their gradients, the learning rate, etc.
          optimizer.step()

          # Update the learning rate.
          scheduler.step()

      # Calculate the average loss over all of the batches.
      avg_train_loss = total_train_loss / len(train_dataloader)            
      
      # Measure how long this epoch took.
      training_time = format_time(time.time() - t0)

      print("")
      print("  Average training loss: {0:.2f}".format(avg_train_loss))
      print("  Training epcoh took: {:}".format(training_time))
          
      # ========================================
      #               Validation
      # ========================================
      # After the completion of each training epoch, measure our performance on
      # our validation set.

      print("")
      print("Running Validation...")

      t0 = time.time()

      # Put the model in evaluation mode--the dropout layers behave differently
      # during evaluation.
      model.eval()

      # Tracking variables 
      total_eval_accuracy = 0
      total_eval_loss = 0
      nb_eval_steps = 0

      # Evaluate data for one epoch
      for batch in validation_dataloader:
          
          # Unpack this training batch from our dataloader. 
          #
          # As we unpack the batch, we'll also copy each tensor to the GPU using 
          # the `to` method.
          #
          # `batch` contains three pytorch tensors:
          #   [0]: input ids 
          #   [1]: attention masks
          #   [2]: labels 
          b_input_ids = batch[0].to(device)
          b_input_mask = batch[1].to(device)
          b_labels = batch[2].to(device)
          
          # Tell pytorch not to bother with constructing the compute graph during
          # the forward pass, since this is only needed for backprop (training).
          with torch.no_grad():        

              # Forward pass, calculate logit predictions.
              # token_type_ids is the same as the "segment ids", which 
              # differentiates sentence 1 and 2 in 2-sentence tasks.
              result = model(b_input_ids, 
                            token_type_ids=None, 
                            attention_mask=b_input_mask,
                            labels=b_labels,
                            return_dict=True)

          # Get the loss and "logits" output by the model. The "logits" are the 
          # output values prior to applying an activation function like the 
          # softmax.
          loss = result.loss
          logits = result.logits
              
          # Accumulate the validation loss.
          total_eval_loss += loss.item()

          # Move logits and labels to CPU
          logits = logits.detach().cpu().numpy()
          label_ids = b_labels.to('cpu').numpy()

          # Calculate the accuracy for this batch of test sentences, and
          # accumulate it over all batches.
          total_eval_accuracy += flat_accuracy(logits, label_ids)
          

      # Report the final accuracy for this validation run.
      avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
      print("  Accuracy: {0:.2f}".format(avg_val_accuracy))

      # Calculate the average loss over all of the batches.
      avg_val_loss = total_eval_loss / len(validation_dataloader)
      
      # Measure how long the validation run took.
      validation_time = format_time(time.time() - t0)
      
      print("  Validation Loss: {0:.2f}".format(avg_val_loss))
      print("  Validation took: {:}".format(validation_time))

      # Record all statistics from this epoch.
      training_stats.append(
          {
              'epoch': epoch_i + 1,
              'Training Loss': avg_train_loss,
              'Valid. Loss': avg_val_loss,
              'Valid. Accur.': avg_val_accuracy,
              'Training Time': training_time,
              'Validation Time': validation_time
          }
      )

  print("")
  print("Training complete!")

  print("Total training took {:} (h:mm:ss)".format(format_time(time.time()-total_t0)))
  return training_stats


# + id="0z0S2Ds-_MST"
def format_training_stats(training_stats):
  # Display floats with two decimal places.
  pd.set_option('precision', 2)

  # Create a DataFrame from our training statistics.
  df_stats = pd.DataFrame(data=training_stats)

  # Use the 'epoch' as the row index.
  df_stats = df_stats.set_index('epoch')

  # A hack to force the column headers to wrap.
  #df = df.style.set_table_styles([dict(selector="th",props=[('max-width', '70px')])])

  # Display the table.
  return df_stats


# + id="P5ODbXZx_WeH"
import matplotlib.pyplot as plt
% matplotlib inline

import seaborn as sns

# Use plot styling from seaborn.
sns.set(style='darkgrid')

# Increase the plot size and font size.
sns.set(font_scale=1.5)

def plot_traing_val_loss(df_stats):
  plt.rcParams["figure.figsize"] = (12,6)

  # Plot the learning curve.
  plt.plot(df_stats['Training Loss'], 'b-o', label="Training")
  plt.plot(df_stats['Valid. Loss'], 'g-o', label="Validation")

  # Label the plot.
  plt.title("Training & Validation Loss")
  plt.xlabel("Epoch")
  plt.ylabel("Loss")
  plt.legend()
  plt.xticks([1, 2, 3, 4])

  plt.show()


# + id="mxdp8B3zFImF"
def prepare_test_dataset(sentences, labels, batch_size=16):
  # Tokenize all of the sentences and map the tokens to thier word IDs.
  input_ids = []
  attention_masks = []

  # For every sentence...
  for sent in sentences:
      # `encode_plus` will:
      #   (1) Tokenize the sentence.
      #   (2) Prepend the `[CLS]` token to the start.
      #   (3) Append the `[SEP]` token to the end.
      #   (4) Map tokens to their IDs.
      #   (5) Pad or truncate the sentence to `max_length`
      #   (6) Create attention masks for [PAD] tokens.
      encoded_dict = tokenizer.encode_plus(
                          sent,                      # Sentence to encode.
                          add_special_tokens = True, # Add '[CLS]' and '[SEP]'
                          max_length = 64,           # Pad & truncate all sentences.
                          pad_to_max_length = True,
                          return_attention_mask = True,   # Construct attn. masks.
                          return_tensors = 'pt',     # Return pytorch tensors.
                    )
      
      # Add the encoded sentence to the list.    
      input_ids.append(encoded_dict['input_ids'])
      
      # And its attention mask (simply differentiates padding from non-padding).
      attention_masks.append(encoded_dict['attention_mask'])

  # Convert the lists into tensors.
  input_ids = torch.cat(input_ids, dim=0)
  attention_masks = torch.cat(attention_masks, dim=0)
  labels = torch.tensor(labels)

  # Create the DataLoader.
  prediction_data = TensorDataset(input_ids, attention_masks, labels)
  prediction_sampler = SequentialSampler(prediction_data)
  prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)

  return prediction_data, prediction_sampler, prediction_dataloader


# + id="tOzQE43xHD4a"
def generate_test_predictions(model, prediction_dataloader):
  # Prediction on test set

  # print('Predicting labels for {:,} test sentences...'.format(len(input_ids)))

  # Put model in evaluation mode
  model.eval()

  # Tracking variables 
  predictions , true_labels = [], []

  # Predict 
  for batch in prediction_dataloader:
    # Add batch to GPU
    batch = tuple(t.to(device) for t in batch)
    
    # Unpack the inputs from our dataloader
    b_input_ids, b_input_mask, b_labels = batch
    
    # Telling the model not to compute or store gradients, saving memory and 
    # speeding up prediction
    with torch.no_grad():
        # Forward pass, calculate logit predictions.
        result = model(b_input_ids, 
                      token_type_ids=None, 
                      attention_mask=b_input_mask,
                      return_dict=True)
    # print(result)
    logits = result.logits

    # Move logits and labels to CPU
    logits = logits.detach().cpu().numpy()
    label_ids = b_labels.to('cpu').numpy()
    
    # Store predictions and true labels
    predictions.append(logits)
    true_labels.append(label_ids)

  print('DONE.')
  return predictions , true_labels


# + id="IZYdjnlHHGjy"
def calculate_accuracy(predictions, true_labels):
  # Combine the results across all batches. 
  flat_predictions = np.concatenate(predictions, axis=0)

  # For each sample, pick the label (0 , 1, 2) with the higher score.
  flat_predictions = np.argmax(flat_predictions, axis=1).flatten()
  # Combine the correct labels for each batch into a single list.
  flat_true_labels = np.concatenate(true_labels, axis=0)

  acc = np.sum(flat_predictions == flat_true_labels) / len(flat_true_labels)
  return acc



# + [markdown] id="jNalEz7X9mzS"
# ## Finetuning with English tweets

# + [markdown] id="1BIGflsd1C-w"
# #### Model Setup

# + colab={"base_uri": "https://localhost:8080/"} id="SAtjToipxxWb" outputId="bc30aeba-5e1f-4c90-fd86-dbd9fcfa0863"
# Load the tokenizer. 
# tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base", do_lower_case=True)
tokenizer = XLMRobertaTokenizer.from_pretrained("xlm-roberta-base", do_lower_case=True)
# Load the model.
model = XLMRobertaForSequenceClassification.from_pretrained(
    "xlm-roberta-base", 
    num_labels = 3, # The number of output labels--3 for binary classification.
                    # You can increase this for multi-class tasks.   
    output_attentions = False, # Whether the model returns attentions weights.
    output_hidden_states = False, # Whether the model returns all hidden-states.
)
# Tell pytorch to run this model on the GPU.
model.cuda()
print('done')

# + id="3m9_Amn17t18"
sentences = en_tweets_df.preprocessed.values
labels = en_tweets_df.sentiment.values

# + [markdown] id="E_baDKX-yK3B"
# #### Fine-tuning the model 

# + colab={"base_uri": "https://localhost:8080/"} id="RJDLLXAC7kkz" outputId="a7ea2ff5-1f8a-44f8-b283-139d4f9a1c69"
input_ids, attention_masks, labels = tokenize_sentences(sentences, labels, tokenizer)

# + colab={"base_uri": "https://localhost:8080/"} id="nV3q9bhg71n1" outputId="1e1141ec-7909-4ab2-eba5-c9530479156b"
train_dataset, val_dataset, train_size, val_size = train_val_split(input_ids, attention_masks, labels)
print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))

# + id="_FQGCkVY9XEb"
train_dataloader, validation_dataloader = get_data_loaders(train_dataset, val_dataset)

# + id="ZeVulGsQ92-v"
optimizer = AdamW(model.parameters(),
                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5
                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.
                )

# + id="KZWUiBWl-IVC"
scheduler = get_lr_scheduler(train_dataloader)

# + colab={"base_uri": "https://localhost:8080/"} id="UQO7fNKZ-M91" outputId="ff061de4-99af-449d-e85a-f9def80c246c"
training_stats = train(model, epochs, train_dataloader, validation_dataloader, scheduler, seed_val=42)

# + colab={"base_uri": "https://localhost:8080/", "height": 204} id="eJjacJXQ-mGN" outputId="b6458b26-516d-4575-ed08-fb07cd4845ad"
df_stats = format_training_stats(training_stats)
df_stats.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 427} id="sug3YhjdEbrL" outputId="28d2a0c0-fc8d-488d-e9c6-b3b0908f5d59"
plot_traing_val_loss(df_stats)

# + colab={"base_uri": "https://localhost:8080/"} id="BYALM2dcOAxc" outputId="4538eaa5-7913-4627-d3c6-a3e9e52bc0c4"
# Create sentence and label lists
sentences = ar_test_df.preprocessed.values
labels = ar_test_df.sentiment.values

prediction_data, prediction_sampler, prediction_dataloader = prepare_test_dataset(sentences, labels)

# + colab={"base_uri": "https://localhost:8080/"} id="BZEi6UMXPYjN" outputId="b1de685c-da56-4297-f79b-3af12adf61fd"
predictions , true_labels = generate_test_predictions(model, prediction_dataloader)

# + colab={"base_uri": "https://localhost:8080/"} id="6nNAplTrPdRt" outputId="67cc03d8-a12c-4ea9-86d7-4e401e56222d"
acc = calculate_accuracy(predictions , true_labels)
print(str('Accuracy: {:04.4f}'.format(acc*100))+'%')

# + [markdown] id="gkt-ZY7hFBbN"
# ## Finetuning with English and Arabic tweets 

# + [markdown] id="5RZkqyZmE8sQ"
# #### Model Setup

# + colab={"base_uri": "https://localhost:8080/"} id="4PpvaUTEFVo3" outputId="002946e2-9094-40c5-f142-6cb12d4b4c46"
train_df = pd.concat([en_tweets_df, ar_train_df], axis=0, ignore_index=True)

print(f"total train tweets {train_df.shape[0]}")
print(f"total test tweets {ar_tweets_df.shape[0]}")

# + colab={"base_uri": "https://localhost:8080/"} id="jVPQEFgbE8sc" outputId="84c2eaf0-0523-4c25-962e-69de46dd8dcd"
# Load the tokenizer. 
tokenizer = XLMRobertaTokenizer.from_pretrained("xlm-roberta-base", do_lower_case=True)

# Load the model.
model = XLMRobertaForSequenceClassification.from_pretrained(
    "xlm-roberta-base", 
    num_labels = 3, # The number of output labels--3 for binary classification.
                    # You can increase this for multi-class tasks.   
    output_attentions = False, # Whether the model returns attentions weights.
    output_hidden_states = False, # Whether the model returns all hidden-states.
)
# Tell pytorch to run this model on the GPU.
model.cuda()
print('done')

# + id="piv9n3wbE8sd"
sentences = train_df.preprocessed.values
labels = train_df.sentiment.values

# + colab={"base_uri": "https://localhost:8080/"} id="Ynlml9g5E8sd" outputId="455ab0fe-f72b-4f29-adca-5bfb3111e953"
input_ids, attention_masks, labels = tokenize_sentences(sentences, labels, tokenizer)

# + colab={"base_uri": "https://localhost:8080/"} id="ntDfxhtoE8se" outputId="534e16cf-8b15-474c-ee6a-3e5b95079447"
train_dataset, val_dataset, train_size, val_size = train_val_split(input_ids, attention_masks, labels)
print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))

# + [markdown] id="o4xdFeKSzgFS"
# #### Fine-tuning the model 

# + id="eL_O3cyIE8se"
train_dataloader, validation_dataloader = get_data_loaders(train_dataset, val_dataset)

# + id="kZ_XcoBBE8se"
optimizer = AdamW(model.parameters(),
                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5
                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.
                )

# + id="kktaavLiE8se"
scheduler = get_lr_scheduler(train_dataloader)

# + [markdown] id="brsnaQL-lWUA"
#

# + colab={"base_uri": "https://localhost:8080/"} id="u0Imzzs1E8se" outputId="3853bc77-5ef2-43b5-ccea-543a87ef68c2"
training_stats = train(model, epochs, train_dataloader, validation_dataloader, scheduler, seed_val=42)

# + colab={"base_uri": "https://localhost:8080/", "height": 204} id="SUHe_AKXE8sf" outputId="0cb0e191-d240-4d36-d529-4d6185d5fd66"
df_stats = format_training_stats(training_stats)
df_stats.head()

# + colab={"base_uri": "https://localhost:8080/", "height": 427} id="LQIjshUqE8sf" outputId="89dc3b88-734d-48ff-f39d-a820431c092a"
plot_traing_val_loss(df_stats)

# + colab={"base_uri": "https://localhost:8080/"} id="SMDGrQr_TZSK" outputId="d5dcd51b-39b0-498f-cf48-742694ab9871"
# Create sentence and label lists
sentences = ar_test_df.preprocessed.values
labels = ar_test_df.sentiment.values

prediction_data, prediction_sampler, prediction_dataloader = prepare_test_dataset(sentences, labels)

# + colab={"base_uri": "https://localhost:8080/"} id="AzZPvDFyTZSP" outputId="8f3a5a40-a1ce-4a6b-ef5a-e6f00c0cf109"
predictions , true_labels = generate_test_predictions(model, prediction_dataloader)

# + colab={"base_uri": "https://localhost:8080/"} id="9NfmUGLVTZSQ" outputId="8f11b044-9bea-4168-8442-f8f2a2c8c329"
acc = calculate_accuracy(predictions , true_labels)
print(str('Accuracy: {:04.4f}'.format(acc*100))+'%')

# + id="tzZQqSWtTb_n"


StarcoderdataJupyterScriptsDedupFiltered