# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
### we start with the liar dataset

import numpy as np
import pandas as pd
# -

#get the dataset
df_train = pd.read_csv('../../../datasets/liar_tweaked/trainvectordata.csv')
df_test = pd.read_csv('../../../datasets/liar_tweaked/testvectordata.csv')
df_valid = pd.read_csv('../../../datasets/liar_tweaked/validvectordata.csv')

X_train=df_train['statement']
X_test=df_test['statement']
Y_train=df_train['label']
Y_test=df_test['label']
X_valid=df_valid['statement']
Y_valid=df_valid['label']

X_test=X_test.append(X_valid, ignore_index=True)

Y_test=Y_test.append(Y_valid, ignore_index=True)

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
#make necessary imports
import numpy as np
import pandas as pd
import itertools
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression


#tfidf
tfidf_vectorizer=TfidfVectorizer(stop_words='english', max_df=0.7)
tfidf_train=tfidf_vectorizer.fit_transform(X_train) 
tfidf_test=tfidf_vectorizer.transform(X_test)  

np.array(tfidf_test.todense().tolist()).shape

tfidf_train = np.array(tfidf_train.todense().tolist())
tfidf_test = np.array(tfidf_test.todense().tolist())

Y_train = np.array(Y_train.tolist())
Y_test = np.array(Y_test.tolist())

# +
import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import *
from tensorflow.keras.models import *

## define variable
n_sample = 10229
text_len = 11915

## create dummy data
# x_train = np.random.uniform(0,1, (n_sample,text_len))
# y_train = np.random.randint(0,2, n_sample)

# print(x_train.shape, y_train.shape)

# print(type(x_train), type(y_train.shape))




# ## expand train dimnesion: pass from 2d to 3d
tfidf_train = np.expand_dims(tfidf_train, axis=1)
tfidf_test = np.expand_dims(tfidf_test, axis=1)
# print(x_train.shape, y_train.shape)

## create model
inp = Input(shape=(1,text_len))
conv2 = Conv1D(filters=128, kernel_size=5, activation='relu', padding='same')(inp)
drop21 = Dropout(0.5)(conv2)
conv22 = Conv1D(filters=64, kernel_size=5, activation='relu', padding='same')(drop21)
drop22 = Dropout(0.5)(conv22)
pool2 = Flatten()(drop22) # this is an option to pass from 3d to 2d
out = Dense(2, activation='softmax')(pool2) # the output dim must be equal to the num of class if u use softmax

model = Model(inp, out)
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
history = model.fit(tfidf_train, Y_train, epochs=100,
                    validation_data=(tfidf_test, Y_test))

# +
### now we have the kaggle dataset

# +
df = pd.read_csv('../../../datasets/kaggle/train.csv')


# -

df

df['content'] = df['title'] + ' ' + df['text']
df=df[df['content']==df['content']]
df=df[df['label']==df['label']]

df

df.to_csv('../../../datasets/kaggle/final.csv')

df=pd.read_csv('../../../datasets/kaggle/final.csv')

df

X=df['content']
Y=df['label']

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.25, random_state=42)

X_test=X_test.reset_index()
Y_test=Y_test.reset_index()
X_train=X_train.reset_index()
Y_train=Y_train.reset_index()

Y_test=Y_test['label']

X_test=X_test['content']
X_train=X_train['content']
Y_train=Y_train['label']

#tfidf
tfidf_vectorizer=TfidfVectorizer(stop_words='english', max_df=0.7)
tfidf_train=tfidf_vectorizer.fit_transform(X_train) 
tfidf_test=tfidf_vectorizer.transform(X_test)

tfidf_test = tfidf_test.toarray()

tfidf_train=tfidf_train.toarray()

Y_train = np.array(Y_train.tolist())
Y_test = np.array(Y_test.tolist())

tfidf_train.shape

# +
import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import *
from tensorflow.keras.models import *

## define variable
n_sample = len(X_train)
text_len = 157102

## create dummy data
# x_train = np.random.uniform(0,1, (n_sample,text_len))
# y_train = np.random.randint(0,2, n_sample)

# print(x_train.shape, y_train.shape)

# print(type(x_train), type(y_train.shape))




# ## expand train dimnesion: pass from 2d to 3d
tfidf_train = np.expand_dims(tfidf_train, axis=1)
tfidf_test = np.expand_dims(tfidf_test, axis=1)
# print(x_train.shape, y_train.shape)

## create model
inp = Input(shape=(1,text_len))
conv2 = Conv1D(filters=128, kernel_size=5, activation='relu', padding='same')(inp)
drop21 = Dropout(0.5)(conv2)
conv22 = Conv1D(filters=64, kernel_size=5, activation='relu', padding='same')(drop21)
drop22 = Dropout(0.5)(conv22)
pool2 = Flatten()(drop22) # this is an option to pass from 3d to 2d
out = Dense(2, activation='softmax')(pool2) # the output dim must be equal to the num of class if u use softmax

model = Model(inp, out)
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
history = model.fit(tfidf_train, Y_train, epochs=100,
                    validation_data=(tfidf_test, Y_test))
# -



StarcoderdataJupyterScriptsDedupFiltered