<jupyter_start><jupyter_text><jupyter_code>!pip install vaderSentiment==3.2.1 requests numpy matplotlib pandas flair
!nvidia-smi
!wget - O politifact_fake.csv https://raw.githubusercontent.com/KaiDMML/FakeNewsNet/master/dataset/politifact_fake.csv
!wget - O politifact_real.csv https://raw.githubusercontent.com/KaiDMML/FakeNewsNet/master/dataset/politifact_real.csv
!ls -l
import os
import json
import numpy as np
import matplotlib.pyplot as plt
from collections import Counter
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import pandas as pd
from flair.embeddings import WordEmbeddings, FlairEmbeddings, DocumentPoolEmbeddings, Sentence, DocumentRNNEmbeddings
from flair.models import TextClassifier
from flair.trainers import ModelTrainer
from flair.datasets import CSVClassificationCorpus
import os
from sklearn.metrics import f1_score, accuracy_score, recall_score, precision_score
from sklearn.model_selection import train_test_split
DATASET_NAME = 'politifact'
DATASET_PATH = './{}'.format(DATASET_NAME)
REAL_DATA_PATH = '{}_real.csv'.format(DATASET_PATH)
FAKE_DATA_PATH = '{}_fake.csv'.format(DATASET_PATH)
fake_arts = pd.read_csv(FAKE_DATA_PATH, na_values=['nan'], keep_default_na=False)
real_arts = pd.read_csv(REAL_DATA_PATH, na_values=['nan'], keep_default_na=False)
def parse_art_data_frame(df):
    return [{'id': id, 'url': url, 'title': title} for id, url, title, tweets in df.values]
    
fake_arts_with_content = parse_art_data_frame(fake_arts)
real_arts_with_content = parse_art_data_frame(real_arts)
fake_data = [(art, 'fake') for art in fake_arts_with_content]
real_data = [(art, 'real') for art in real_arts_with_content]
all_data = fake_data + real_data
X = [x for x,y in all_data]
y = [y for x,y in all_data]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

train_data = list(zip(X_train, y_train))
test_data = list(zip(X_test, y_test))
print(len(train_data))
print(len(test_data))
print(len(train_data) + len(test_data))
print(len(fake_data) + len(real_data))
print(len([y for y in y_train if y == 'fake']))
print(len([y for y in y_test if y == 'fake']))

print(len([y for y in y_train if y == 'real']))
print(len([y for y in y_test if y == 'real']))
def clear_text(text):
    return ' '.join(text.split())

class Classifier():
    def __init__(self, classifier):
        self.classifier = classifier
        
    def predict(self, text):
        text = clear_text(text)
        sentence = Sentence(text)
        self.classifier.predict(sentence)
        return sentence.labels[0]

def transform_data(data):
    return [{'label': label, 'text': clear_text(x)} for x, label in data]

def save_data(data, data_folder = '.'):
    if not os.path.exists(data_folder):
        os.makedirs(data_folder)
    data = transform_data(data)
    frame_data = pd.DataFrame(data)
    train_path = '{}/train.csv'.format(data_folder)
    test_path = '{}/test.csv'.format(data_folder)
    dev_path = '{}/dev.csv'.format(data_folder)
    frame_data.iloc[0:int(len(data)*0.8)].to_csv(train_path, sep='\t', index = False, header = False)
    frame_data.iloc[int(len(data)*0.8):int(len(data)*0.9)].to_csv(test_path, sep='\t', index = False, header = False)
    frame_data.iloc[int(len(data)*0.9):].to_csv(dev_path, sep='\t', index = False, header = False)

def load_corpus(data_folder = '.'):
    column_name_map = {1: "text", 0: "label"}
    return CSVClassificationCorpus(data_folder,
                                     column_name_map,
                                     delimiter='\t',
                                  test_file='test.csv',
                                  dev_file='dev.csv',
                                  train_file='train.csv')
    
def train_classifier(corpus, model_folder = '.', max_epochs = 1):
    label_dict = corpus.make_label_dictionary()

    word_embeddings = [
        WordEmbeddings('glove'),
        FlairEmbeddings('news-forward-fast'),
        FlairEmbeddings('news-backward-fast')
    ]

    document_embeddings = DocumentRNNEmbeddings(word_embeddings,
                                                hidden_size=512,
                                                reproject_words=True,
                                                reproject_words_dimension=256)

    classifier = TextClassifier(document_embeddings,
                                label_dictionary=label_dict)

    trainer = ModelTrainer(classifier, corpus)

    trainer.train(model_folder, max_epochs=max_epochs)
    
    return TextClassifier.load('{}/best-model.pt'.format(model_folder))
    
def train_model(train_data,
               data_folder = '.',
               model_folder = '.',
               max_epochs=1
               ):
    save_data(train_data, data_folder)
    corpus = load_corpus(data_folder)
    classifier = train_classifier(corpus, model_folder, max_epochs)
    return Classifier(classifier)

def calculate_metrics(y_true, y_pred, pos_label = 'fake'):
    acc = accuracy_score(y_true, y_pred)
    precision = precision_score(y_true, y_pred, pos_label=pos_label)
    recall = recall_score(y_true, y_pred, pos_label=pos_label)
    f1 = f1_score(y_true, y_pred, pos_label=pos_label)
    return acc, precision, recall, f1

def validate_model(test_data, classifier):
    y_true = [label for x, label in test_data]
    y_pred = [classifier.predict(x).value for x, label in test_data]
    acc, precision, recall, f1 = calculate_metrics(y_true, y_pred)
    print("acc: ", acc)
    print("precision: ", precision)
    print("recall: ", recall)
    print("f1: ", f1)
    return acc, precision, recall, f1

def make_test(train_data, test_data, data_folder, model_folder, max_epochs):
    classifier = train_model(train_data, data_folder, model_folder, max_epochs)
    validate_model(test_data, classifier)
train_content = [(x, label) for x, label in train_data] 
test_content = [(x, label) for x, label in test_data]
train_title = [(x['title'], label) for x, label in train_content]
test_title = [(x['title'], label) for x, label in test_content]
print(len([x for x, label in train_title if x == '']))
print(len([x for x, label in test_title if x == '']))
train_title = [(x, label) for x, label in train_title if x != ''] 
test_title = [(x, label) for x, label in test_title if x != '']
print(len(train_title))
print(len(test_title))
path = './test_csv/title1'
make_test(train_title, test_title, path, path, 1)
path = './test_csv/title10'
make_test(train_title, test_title, path, path, 10)
train_url = [(x['url'], label) for x, label in train_content] 
test_url = [(x['url'], label) for x, label in test_content]
print(len([x for x, label in train_url if x == '']))
print(len([x for x, label in test_url if x == '']))
train_url = [(x, label) for x, label in train_url if x != ''] 
test_url = [(x, label) for x, label in test_url if x != '']
print(len(train_url))
print(len(test_url))
path = './test_csv/url1'
make_test(train_url, test_url, path, path, 1)
path = './test_csv/url10'
make_test(train_url, test_url, path, path, 10)
train_mix = [(x['url'] + ', ' + x['title'], label) for x, label in train_content] 
test_mix = [(x['url'] + ', ' + x['title'], label) for x, label in test_content]
print(len([x for x, label in train_mix if x == '']))
print(len([x for x, label in test_mix if x == '']))
train_mix = [(x, label) for x, label in train_mix if x != ''] 
test_mix = [(x, label) for x, label in test_mix if x != '']
print(len(train_mix))
print(len(test_mix))
path = './test_csv/mix1'
make_test(train_mix, test_mix, path, path, 1)
path = './test_csv/mix10'
make_test(train_mix, test_mix, path, path, 10)
path = './test_csv/title50'
make_test(train_title, test_title, path, path, 50)
path = './test_csv/url50'
make_test(train_url, test_url, path, path, 50)
path = './test_csv/mix50'
make_test(train_mix, test_mix, path, path, 50)<jupyter_output><empty_output>