# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### Import the required libraries or modules

import nltk
import random
import pickle
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import re
import os

# ### Optional

# +
#nltk.download('punkt')

# +
#nltk.download('averaged_perceptron_tagger')
# -

# ### Read training data (in separate folders, positive or negative) from local files

files_pos = os.listdir('data/train/pos')
files_pos = [open('data/train/pos/'+f, 'r', encoding="utf8").read() for f in files_pos]
files_neg = os.listdir('data/train/neg')
files_neg = [open('data/train/neg/'+f, 'r', encoding="utf8").read() for f in files_neg]

# ### Creating bag of words for creating featureset

# +
all_words = []
documents = []

stop_words = list(set(stopwords.words('english')))

#  j is adject, r is adverb, and v is verb
#allowed_word_types = ["J","R","V"]
allowed_word_types = ["J"]

for p in  files_pos:
    
    # create a list of tuples where the first element of each tuple is a review
    # the second element is the "pos" label
    documents.append( (p, "pos") )
    
    # remove punctuations
    cleaned = re.sub(r'[^(a-zA-Z)\s]','', p)
    
    # tokenize 
    tokenized = word_tokenize(cleaned)
    
    # remove stopwords 
    stopped = [w for w in tokenized if not w in stop_words]
    
    # parts of speech tagging for each word 
    pos = nltk.pos_tag(stopped)
    
    # make a list of  all adjectives identified by the allowed word types list above
    for w in pos:
        if w[1][0] in allowed_word_types:
            all_words.append(w[0].lower())
# -

for p in files_neg:
    # create a list of tuples where the first element of each tuple is a review
    # the second element is the "neg" label
    documents.append( (p, "neg") )
    
    # remove punctuations
    cleaned = re.sub(r'[^(a-zA-Z)\s]','', p)
    
    # tokenize 
    tokenized = word_tokenize(cleaned)
    
    # remove stopwords 
    stopped = [w for w in tokenized if not w in stop_words]
    
    # parts of speech tagging for each word 
    neg = nltk.pos_tag(stopped)
    
    # make a list of  all adjectives identified by the allowed word types list above
    for w in neg:
        if w[1][0] in allowed_word_types:
            all_words.append(w[0].lower())

# ### Saving the labeled data 

save_documents = open("pickled_data/documents.pickle","wb")
pickle.dump(documents, save_documents)
save_documents.close()

all_words = nltk.FreqDist(all_words)
word_features = list(all_words.keys())[:5000]

# ### Saving the bag of words which will be used as featureset

save_word_features = open("pickled_data/word_features5k.pickle","wb")
pickle.dump(word_features, save_word_features)
save_word_features.close()

StarcoderdataJupyterScriptsDedupFiltered