<jupyter_start><jupyter_text>What files do I need?You'll need train.csv, test.csv and sample_submission.csv.What should I expect the data format to be?Each sample in the train and test set has the following information:    The text of a tweet    A keyword from that tweet (although this may be blank!)    The location the tweet was sent from (may also be blank)What am I predicting?You are predicting whether a given tweet is about a real disaster or not. If so, predict a 1. If not, predict a 0.Files    train.csv - the training set    test.csv - the test set    sample_submission.csv - a sample submission file in the correct formatColumns    id - a unique identifier for each tweet    text - the text of the tweet    location - the location the tweet was sent from (may be blank)    keyword - a particular keyword from the tweet (may be blank)    target - in train.csv only, this denotes whether a tweet is about a real disaster (1) or not (0)<jupyter_code>import re # for regular expressions
import pandas as pd 
pd.set_option("display.max_colwidth", 200)
import numpy as np 
import matplotlib.pyplot as plt 
import seaborn as sns
import string
import nltk # for text manipulation
from nltk.stem.porter import *
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from tqdm import tqdm
#from gensim.models.doc2vec import LabeledSentence
#import gensim
from sklearn.linear_model import LogisticRegression
from scipy import stats 
from sklearn import metrics 
from sklearn.metrics import mean_squared_error,mean_absolute_error, make_scorer,classification_report,confusion_matrix,accuracy_score,roc_auc_score,roc_curve
from sklearn.model_selection import train_test_split,cross_val_score,KFold
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from sklearn.naive_bayes import BernoulliNB
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
#from xgboost import XGBClassifier
#import xgboost as xgb
import warnings 
warnings.filterwarnings("ignore")

%matplotlib inline
import sys
stdout = sys.stdout

# some functions that mess up sys.stdout

sys.stdout = stdout
IS_CREDIBLE_DISASTER = 0 # init

# a generator creates fake data 

# a discriminator compares that to real data 

# each is scored and improved until generator data is above acceptable threshhold 
import tqdm 
from tqdm import tqdm

from keras.preprocessing.text import text_to_word_sequence
# define the text 
text = train_data
# tokenizing the text 
tokens = text_to_word_sequence(text)
print(tokens[0])

# Load CSV (using python)
import csv
import numpy
import pandas as pd
from textblob import TextBlob
filename = '/Users/grahamwaters/Documents/GitHub Main Repository/KAGGLE_COMPETITIONS/DISASTERS/test.csv'
#reader = csv.reader(raw_data, delimiter=',', quoting=csv.QUOTE_NONE)
df_test = pd.read_csv(filename)
filename = '/Users/grahamwaters/Documents/GitHub Main Repository/KAGGLE_COMPETITIONS/DISASTERS/train.csv'
#reader = csv.reader(raw_data, delimiter=',', quoting=csv.QUOTE_NONE)
df_train = pd.read_csv(filename)
df_train
train_data = pd.array(df_train['text'])
test_data = pd.array(df_test['text'])
test_data
# for the train data set tweets are under header: 'text' and there is another header: 'target' which shows legit or illegit
#merge = train_data.append(test_data,ignore_index = True, sort = False)


import pandas as pd
import re
from string import punctuation
def preprocess_text(text):
    text = text.lower()  # Lowercase text
    text = re.sub(f"[{re.escape(punctuation)}]", "", text)  # Remove punctuation
    text = " ".join(text.split())  # Remove extra spaces, tabs, and new lines
    return text

from textblob import Word # for lemmatization
import nltk

def blobber(df,text):
    text = preprocess_text(text)
    # pass an entry # and a tweet, it disects it and returns a df 
    blob = TextBlob(text)
    
    # lemmatization

    wordlist = blob.words
    upd_wordlist = []
    for w in wordlist:
        upd_wordlist.append(w.lemmatize())
    #print(text[0:10])
    #print('polarity_score:',blob.polarity)
    #print('sentiment_score:',blob.sentiment)
    #print('words_score:',blob.words)
    df[str(text)] = {
        'tags': blob.tags,
        'phrases':blob.noun_phrases,
        'sentiment':blob.sentiment,
        'words':blob.words,
        'lemmatized_words':upd_wordlist,
        'sentences':blob.sentences, # sentences
        'tweet_length':len(str(text)) # length of tweet
    }
    
    return df
import tqdm
from tqdm import tqdm

df_test = {}
print("building test data ...")
for tweet_text in tqdm(test_data):
    df_test = blobber(df_test,tweet_text)

df_training = {}
print("building training data ...")
for tweet_text in tqdm(train_data):
    df_training = blobber(df_training,tweet_text)

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
bow_vectorizer = CountVectorizer(max_df=0.90, min_df=2, max_features=1000, stop_words='english')
bow = bow_vectorizer.fit_transform(df_training['text'])<jupyter_output><empty_output><jupyter_text>now build model based on the train set which has the answers already.  Logistic Regression Modeling<jupyter_code>train_bow = bow[:]<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup