<jupyter_start><jupyter_text><jupyter_code>### importing drive
from google.colab import drive
drive.mount('/content/gdrive')
# !unzip '/content/gdrive/MyDrive/BCG/stumbleupon.zip' -d '/content/gdrive/MyDrive/BCG'
### libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
### Config File 
train = '/content/gdrive/MyDrive/BCG/train.tsv'
test = '/content/gdrive/MyDrive/BCG/test.tsv'
sample = '/content/gdrive/MyDrive/BCG/sampleSubmission.csv'
train_df = pd.read_csv(train,sep='\t')
train_df.head(5)
test_df = pd.read_csv(test,sep='\t')
test_df.head(5)
### Sample Submission
sampleSub = pd.read_csv(sample)
sampleSub.head(5)
xtrain_df  = pd.DataFrame({
    'boilerplate': train_df.boilerplate.values,
    'label': train_df.label.values
})
xtrain_df.head(5)
xtrain_df.shape
xtrain_df.isnull().sum()
plt.style.use('dark_background')
sns.countplot(x='label',data=xtrain_df)
plt.xlabel('Value')
plt.ylabel('label')
plt.show()
X = []
for items in xtrain_df['boilerplate']:
    X.append(items)
from wordcloud import  WordCloud ,STOPWORDS
commonWord = ' '.join(X)
wordcloud = WordCloud(stopwords=STOPWORDS,height=2500,width=3000).generate(commonWord)
plt.figure(1,figsize=(8, 8))
plt.imshow(wordcloud)
plt.axis('off')
plt.show()
del X, wordcloud
from sklearn.model_selection import train_test_split
X = xtrain_df.boilerplate.values
Y = xtrain_df.label.values
x_train,x_val,y_train,y_val = train_test_split(X,Y, test_size = 0.1 , random_state = 2021)
import torch
if torch.cuda.is_available():       
    device = torch.device("cuda")
    print(torch.cuda.get_device_name(0))
!pip install transformers
import string<jupyter_output><empty_output><jupyter_text>Since I can't use greater then 512 len so I decided to preprocess the else it is not needed for the Bert model<jupyter_code>import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
en_stops = stopwords.words('english')
def remove_stop(text):
  return ' '.join([i for i in text.split() if i not in en_stops])
### Simple Preprocessing 
def processed_word(text):
  text = remove_stop(text)
  return ''.join([i for i in text if i not in string.punctuation ])
a = "This is for the intern"
print("Not processed :" ,a)
print("processed word :" ,processed_word(a))
### using based berttokenizer 
from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',do_lower_case = True)
Max_len = 325
def preprocess_from_bert(data):
  input_ids = []
  attention_mask = []

  for item in data:
    token = tokenizer.encode_plus(
        text = processed_word(item),
        add_special_tokens=True,
        max_length = Max_len,
        # padding = 'max_length',
        pad_to_max_length = True,
        truncation = True,
        return_attention_mask =True,
    )
    input_ids.append(token.get('input_ids') )
    attention_mask.append(token.get('attention_mask'))
    
  input_ids = torch.tensor(input_ids)
  attention_mask = torch.tensor(attention_mask)

  return input_ids , attention_mask
train_inputs,train_masks = preprocess_from_bert(x_train)
val_inputs,val_masks = preprocess_from_bert(x_val)
train_masks.shape
##### DataLoader for train and validation dataset
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
train_labels = torch.tensor(y_train)
val_labels = torch.tensor(y_val)
batch_size = 32
train_data = TensorDataset(train_inputs, train_masks, train_labels)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
val_data = TensorDataset(val_inputs, val_masks, val_labels)
val_sampler = SequentialSampler(val_data)
val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=batch_size)

import torch
import torch.nn as nn
from transformers import BertModel
import torch.nn.functional as F<jupyter_output><empty_output><jupyter_text>**Model**<jupyter_code>#### using pretrained model for classification
import torch
import torch.nn as nn
from transformers import BertModel

# Create the BertClassfier class
class BertClassifier(nn.Module):
    def __init__(self, freeze_bert=False):
        super(BertClassifier, self).__init__()
        self.bert = BertModel.from_pretrained('bert-base-uncased')
        self.fn = nn.Linear(768,80)
        self.drop = nn.Dropout(0.2)
        self.fn1 = nn.Linear(80,2)
        if freeze_bert:
            for param in self.bert.parameters():
                param.requires_grad = False
        
    def forward(self, input_ids, attention_mask):
        outputs = self.bert(input_ids=input_ids,
                            attention_mask=attention_mask)
        last_hidden_state_cls = outputs[0][:, 0, :]
        x = F.relu(self.fn(last_hidden_state_cls))
        x = self.drop(x)
        logits = self.fn1(x)
        return logits
from transformers import AdamW, get_linear_schedule_with_warmup

def initialize_model(epochs):
    bert_classifier = BertClassifier(freeze_bert=False)
    bert_classifier.to(device)
    optimizer = AdamW(bert_classifier.parameters(),
                      lr=5e-5,    # Default learning rate
                      eps=1e-8    # Default epsilon value
                      )
    total_steps = len(train_dataloader) * epochs
    scheduler = get_linear_schedule_with_warmup(optimizer,
                                                num_warmup_steps=0, 
                                                num_training_steps=total_steps)
    return bert_classifier, optimizer, scheduler
import random
import time
loss_fn = nn.CrossEntropyLoss()

def set_seed(seed_value=42):
    random.seed(seed_value)
    np.random.seed(seed_value)
    torch.manual_seed(seed_value)
    torch.cuda.manual_seed_all(seed_value)

def train(model, train_dataloader, val_dataloader=None, epochs=2, evaluation=False):
    for epoch_i in range(epochs):
        print(f"{'Epoch':^7} | {'Batch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}")
        print("-"*70)
        t0_epoch, t0_batch = time.time(), time.time()
        total_loss, batch_loss, batch_counts = 0, 0, 0
        model.train()
        for step, batch in enumerate(train_dataloader):
            batch_counts +=1
            b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)
            model.zero_grad()
            logits = model(b_input_ids, b_attn_mask)
            loss = loss_fn(logits, b_labels)
            batch_loss += loss.item()
            total_loss += loss.item()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optimizer.step()
            scheduler.step()
            if (step % 20 == 0 and step != 0) or (step == len(train_dataloader) - 1):
                time_elapsed = time.time() - t0_batch
                print(f"{epoch_i + 1:^7} | {step:^7} | {batch_loss / batch_counts:^12.6f} | {'-':^10} | {'-':^9} | {time_elapsed:^9.2f}")
                batch_loss, batch_counts = 0, 0
                t0_batch = time.time()
        avg_train_loss = total_loss / len(train_dataloader)

        print("-"*70)
        if evaluation == True:
            val_loss, val_accuracy = evaluate(model, val_dataloader)
            time_elapsed = time.time() - t0_epoch
            
            print(f"{epoch_i + 1:^7} | {'-':^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}")
            print("-"*70)

def evaluate(model, val_dataloader):
    model.eval()
    val_accuracy = []
    val_loss = []
    for batch in val_dataloader:
        b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)
        with torch.no_grad():
            logits = model(b_input_ids, b_attn_mask)
        loss = loss_fn(logits, b_labels)
        val_loss.append(loss.item())
        preds = torch.argmax(logits, dim=1).flatten()
        accuracy = (preds == b_labels).cpu().numpy().mean() * 100
        val_accuracy.append(accuracy)
    val_loss = np.mean(val_loss)
    val_accuracy = np.mean(val_accuracy)

    return val_loss, val_accuracy
### training the model
set_seed(42)    # Set seed for reproducibility
bert_classifier, optimizer, scheduler = initialize_model(epochs=2)
train(bert_classifier, train_dataloader, val_dataloader, epochs=2, evaluation=True)
def evaluate_roc(probs, y_true):
    preds = probs[:, 1]
    fpr, tpr, threshold = roc_curve(y_true, preds)
    roc_auc = auc(fpr, tpr)
    print(f'AUC: {roc_auc:.4f}')
    y_pred = np.where(preds >= 0.5, 1, 0)
    accuracy = accuracy_score(y_true, y_pred)
    print(f'Accuracy: {accuracy*100:.2f}%')
    plt.title('Receiver Operating Characteristic')
    plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
    plt.legend(loc = 'lower right')
    plt.plot([0, 1], [0, 1],'r--')
    plt.xlim([0, 1])
    plt.ylim([0, 1])
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    plt.show()
### prediction on val dataset
probs = bert_predict(bert_classifier, val_dataloader)
evaluate_roc(probs, y_val)
preds = probs[:, 1]
preds.shape
from sklearn.metrics import precision_recall_curve,confusion_matrix,precision_score,recall_score,f1_score
# import plotly.express as px
# import plotly.graph_objects as go
precision,recall,thresolds = precision_recall_curve(y_val,preds)
y_pred = np.where(preds >= 0.5, 1, 0)
print("Precision: %f "%precision_score(y_val, y_pred))
print("="*40)
print("Recall: %f "%recall_score(y_val, y_pred))
print("="*40)
print("F1: %f"% f1_score(y_val, y_pred))
print("="*40)
print("Let's see the confuision matrix:\n",confusion_matrix(y_val, y_pred))
print("="*40)
plt.style.use('ggplot')
conf_matrix = confusion_matrix(y_val, y_pred)
plt.figure(figsize=(10,8))
sns.heatmap(conf_matrix,cmap="YlGnBu")
plt.title('Confusion Matrix')
plt.show()
# fig = px.area(
#     x=recall, y=precision,
#     title=f'Precision-Recall Curve (AUC={auc(fpr, tpr):.4f})',
#     labels=dict(x='Recall', y='Precision'),
#     width=700, height=500
# )
# fig.add_shape(
#     type='line', line=dict(dash='dash'),
#     x0=0, x1=1, y0=1, y1=0
# )
# fig.update_yaxes(scaleanchor="x", scaleratio=1)
# fig.update_xaxes(constrain='domain')

# fig.show()

# conf_matrix = confusion_matrix(y_val, y_pred)
# trace1 = go.Heatmap(z = conf_matrix  ,x = ["0 (pred)","1 (pred)"],
#                         y = ["0 (true)","1 (true)"],xgap = 2, ygap = 2, 
#                         showscale  = False, )
# fig = go.Figure(trace1)
# fig.show()
#### Loading Test DataSet
def bert_predict(model, test_dataloader):
    model.eval()
    all_logits = []
    for batch in test_dataloader:
        b_input_ids, b_attn_mask = tuple(t.to(device) for t in batch)[:2]
        with torch.no_grad():
            logits = model(b_input_ids, b_attn_mask)
        all_logits.append(logits)
    all_logits = torch.cat(all_logits, dim=0)
    probs = F.softmax(all_logits, dim=1).cpu().numpy()
    return probs
test_inputs, test_masks = preprocess_from_bert(test_df.boilerplate.values)

test_dataset = TensorDataset(test_inputs, test_masks)
test_sampler = SequentialSampler(test_dataset)
test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=32)
probs = bert_predict(bert_classifier, test_dataloader)

threshold = 0.6
preds = np.where(probs[:, 1] > threshold, 1, 0)

print("Number of evergreen text:", preds.sum())
sampleSub['label'] = preds
sns.countplot(x='label',data=sampleSub)
plt.xlabel('Classes')
plt.ylabel('Number of values predicted')
plt.show()
sampleSub.head(5)
sampleSub.set_index('urlid',inplace=True)
sampleSub.to_csv('finalSubmission.csv')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup