<jupyter_start><jupyter_text><jupyter_code>gpu_info = !nvidia-smi
gpu_info = '\n'.join(gpu_info)
if gpu_info.find('failed') >= 0:
  print('Not connected to a GPU')
else:
  print(gpu_info)
from psutil import virtual_memory
ram_gb = virtual_memory().total / 1e9
print('Your runtime has {:.1f} gigabytes of available RAM\n'.format(ram_gb))

if ram_gb < 20:
  print('Not using a high-RAM runtime')
else:
  print('You are using a high-RAM runtime!')
!pip install datasets
!pip install transformers
!pip3 install torchinfo
import transformers 
import  matplotlib.pyplot as plt
from datasets import load_dataset
import numpy as np 
import pandas as pd 
from pprint import pprint
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, f1_score, precision_score, recall_score,roc_auc_score,accuracy_score
from sklearn.metrics import plot_confusion_matrix,classification_report
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from datasets import load_metric<jupyter_output><empty_output><jupyter_text>Datasets<jupyter_code>raw_datasets = load_dataset("glue","sst2")

raw_datasets
print(raw_datasets['train'])
print(dir(raw_datasets['train']))
type(raw_datasets['train'])
raw_datasets['train'].data
raw_datasets['train'][0]
raw_datasets['train'][50000:50003]
raw_datasets['train'].features
len(raw_datasets['train'])
raw_datasets['train'].column_names
raw_datasets['train']['sentence'][:5]<jupyter_output><empty_output><jupyter_text>From Datasets to DataFrame<jupyter_code>raw_datasets.set_format(type='pandas')
tweet_data = raw_datasets['train'][:]

int2str_label = raw_datasets['train'].features['label'].names
int2str_mapping = {k:v for k,v in enumerate(int2str_label)}
int2str_mapping
%%timeit
tweet_data['label_name']= tweet_data['label'].map(int2str_mapping)
tweet_data.head()
tweet_data['label_name'].value_counts(ascending=True).plot.barh()
tweet_data['tweet_length']= tweet_data['sentence'].apply(lambda x: len(x.split(' ')))
tweet_data.boxplot("tweet_length", by="label_name", color="black",grid=False, showfliers=False)
tweet_data.groupby(['label_name'])['tweet_length'].describe()
raw_datasets.reset_format()<jupyter_output><empty_output><jupyter_text>From Text to Tokenization<jupyter_code>from transformers import AutoTokenizer
checkpoint = 'distilbert-base-uncased'
tokenizer = AutoTokenizer.from_pretrained(checkpoint)<jupyter_output><empty_output><jupyter_text>Text Tokens to Integer Ids<jupyter_code>tokenized_sentences = tokenizer(raw_datasets['train'][0:3]['sentence'])
pprint(tokenized_sentences)<jupyter_output>{'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],
 'input_ids': [[101, 5342, 2047, 3595, 8496, 2013, 1996, 18643, 3197, 102],
               [101,
                3397,
                2053,
                15966,
                1010,
                2069,
                4450,
                2098,
                18201,
                2015,
                102],
               [101,
                2008,
                7459,
                2049,
                3494,
                1998,
                10639,
                2015,
                2242,
                2738,
                3376,
                2055,
                2529,
                3267,
                102]]}<jupyter_text>Integer Ids to Tokens<jupyter_code>back2tokens = tokenizer.convert_ids_to_tokens(tokenized_sentences['input_ids'][0])
back2tokens
print(tokenizer.convert_tokens_to_string(back2tokens))<jupyter_output>[CLS] hide new secretions from the parental units [SEP]<jupyter_text>Tokenization the entire dataset<jupyter_code>def tokenize_fn(batch):
  return tokenizer(batch['sentence'], truncation=True,padding=True)

tokenized_dataset = raw_datasets.map(tokenize_fn, batched=True)
tokenized_dataset
pprint(tokenize_fn(raw_datasets['train'][:3]))<jupyter_output>{'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0],
                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]],
 'input_ids': [[101,
                5342,
                2047,
                3595,
                8496,
                2013,
                1996,
                18643,
                3197,
                102,
                0,
                0,
                0,
                0,
                0],
               [101,
                3397,
                2053,
                15966,
                1010,
                2069,
                4450,
                2098,
                18201,
                2015,
                102,
                0,
                0,
                0,
                0],
               [101,
                2008,
                7459,
                2049,
                3494,
                1998,
                10639,
          [...]<jupyter_text>Fine Tuning- Getting the pre_trained model- All the parameters will be trained with a very slow learning rate.   Training Arguments<jupyter_code>from transformers import TrainingArguments
from transformers import training_args
batch_size= 64
logging_steps = len(tokenized_dataset["train"]) // batch_size
model_name = "airline_tweet_analysis_model"
training_args = TrainingArguments(
    'airline_tweet_lp',
    evaluation_strategy='epoch',
    num_train_epochs=2,
    learning_rate=2e-5,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    save_strategy='epoch',
    logging_steps=logging_steps,
    log_level="error",
    push_to_hub=False,
    disable_tqdm=False
    )
from transformers import AutoModelForSequenceClassification
import torch
model_ckpt =  'distilbert-base-uncased'
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
num_labels=2


model = (AutoModelForSequenceClassification.from_pretrained(model_ckpt, num_labels=num_labels).to(device))
model
type(model)
#from torchinfo import summary
#summary(model,input_size=(batch_size,512), dtypes=['torch.IntTensor'],device='cpu')
#summary(model)
params_before = []
for name, p in model.named_parameters():
  print(name,p.shape)
  params_before.append(p.detach().cpu().numpy())
def compute_metrics(pred):
  labels=pred.label_ids
  preds = pred.predictions.argmax(-1)
  f1 = f1_score(labels,preds, average="weighted"),
  acc = accuracy_score(labels,preds)
  return {"accuracy":acc,"f1":f1}


metric = load_metric("glue","sst2")
metric.compute(predictions=[1,0,1], references=[1,0,1])<jupyter_output><empty_output><jupyter_text>Trainer<jupyter_code>from transformers import Trainer 
trainer = Trainer(model=model, args=training_args,
                  compute_metrics=compute_metrics,
                  train_dataset=tokenized_dataset['train'],
                  eval_dataset = tokenized_dataset['validation'],
                  tokenizer=tokenizer)
trainer.train()
pred_outputs = trainer.predict(tokenized_dataset['validation'])
pred_outputs.metrics
y_preds = np.argmax(pred_outputs.predictions,axis=1)
y_valid = np.array(tokenized_dataset['validation']["label"])
labels = tokenized_dataset["train"].features["label"].names
cm = confusion_matrix(y_valid, y_preds, labels=[0,1])
disp = ConfusionMatrixDisplay(confusion_matrix=cm,
                             display_labels=['negative', 'positive'])
disp.plot()
plt.grid(False)
plt.show()<jupyter_output><empty_output><jupyter_text>Saving the model<jupyter_code>trainer.save_model('my_saved_model')
!ls
from transformers import pipeline
newmodel = pipeline('text-classification',model='my_saved_model',device=0)
newmodel('this airline does not give any good meal')
newmodel('i think this airline would have been good if they had inflight entertainement')
newmodel('this flight was peace')
model
!cat my_saved_model/config.json
import json 
config_path = 'my_saved_model/config.json'
with open(config_path) as f:
  j = json.load(f)

j['id2label']={0: 'negative', 1: 'positive'}

with open(config_path,'w') as f:
  json.dump(j,f,indent=2)
!cat my_saved_model/config.json
newmodel = pipeline('text-classification',model='my_saved_model',device=0)
newmodel('this airline does not give any good meal')
params_after = []
for name,p in model.named_parameters():
  params_after.append(p.detach().cpu().numpy())
for p1,p2 in zip(params_before,params_after):
  print(np.sum(np.abs(p1-p2)))<jupyter_output>4295.8564
22.397516
0.42992195
0.37554002
315.29938
0.44935855
310.2603
0.0003125943
314.87762
0.36486322
294.90582
0.28997904
0.4074064
0.2932126
1264.9528
1.5678885
1179.0743
0.2436764
0.39350107
0.24740562
312.3826
0.38964438
307.3603
0.0002814801
277.76935
0.27510706
259.61444
0.23003757
0.36502448
0.22532144
1265.5698
1.4621453
1131.4347
0.22013886
0.38440046
0.23639287
314.73383
0.3718079
316.68652
0.0002969986
278.59503
0.25877658
271.8359
0.22087382
0.3821654
0.22072417
1320.4331
1.5025897
1137.2251
0.2309854
0.39983714
0.22972032
347.77847
0.44709066
348.8623
0.00033897278
319.97488
0.2544848
312.5946
0.24171343
0.3905411
0.2387199
1357.4976
1.6211772
1159.6282
0.2905829
0.40575072
0.2863285
349.1933
0.45537767
353.37854
0.00024720648
299.25183
0.2886273
321.3755
0.3158031
0.39768344
0.3406638
1339.9982
1.5978796
1068.1765
0.2795648
0.41227633
0.28192323
345.38422
0.4643112
341.55566
0.00011034015
323.89346
0.27253997
355.9033
0.2883676
0.4718718
0.34186974
1341.129
1.6796286
[...]