# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
import os
import re
import warnings
warnings.filterwarnings("ignore")

# +
X_train = pd.read_csv("train.csv")
X_test = pd.read_csv('test.csv')

X_train = X_train.drop_duplicates(subset='text')


X_train = X_train[['text', 'target']]
X_test = X_test[['text']]

# +
REPLACE_NO_SPACE = re.compile("(\.)|(\;)|(\:)|(\!)|(\?)|(\,)|(\")|(\()|(\))|(\[)|(\])|(\d+)")
REPLACE_WITH_SPACE = re.compile("(<br\s*/><br\s*/>)|(\-)|(\/)")
NO_SPACE = ""
SPACE = " "

def preprocess_reviews(reviews):
    
    reviews = [REPLACE_NO_SPACE.sub(NO_SPACE, line.lower()) for line in reviews]
    reviews = [REPLACE_WITH_SPACE.sub(SPACE, line) for line in reviews]
    
    return reviews

X_train = preprocess_reviews(X_train)
X_test = preprocess_reviews(X_test)
# -

X_train



StarcoderdataJupyterScriptsDedupFiltered