<jupyter_start><jupyter_text><jupyter_code>! pip install nltk
! python -m textblob.download_corpora
! pip install -U textblob
pip install --upgrade gensim
import csv
import pandas as pd
import spacy
import nltk
import gensim
import gensim.downloader

from textblob.classifiers import NaiveBayesClassifier
from textblob import TextBlob
from collections import Counter
from nltk.corpus import stopwords
from gensim.test.utils import common_texts
from gensim.models import Word2Vec
from gensim.models import KeyedVectors

nltk.download('stopwords')
glove_vector = gensim.downloader.load('glove-twitter-25')
raw_reviews = []
reviews_filename = '/content/drive/MyDrive/reviews.csv.txt'
with open(reviews_filename, 'r') as reviews_csvfile:
    csvreader = csv.reader(reviews_csvfile)
    next(csvreader)
    next(csvreader)
    for i in range(10397):
      row = next(csvreader)
      if (len(row) >= 5):
        row.pop(0)
        row.pop(-1)
        row[2] = ''.join(row[2].split())
        if row[2].replace('.', '', 1).isdigit():
          row[2] = float(row[2])
        row[1] = row[1].rstrip()
        if not row[1] == "" and isinstance(row[2], float):
          raw_reviews.append(row)
table = pd.DataFrame(data = raw_reviews, columns = ['ID', 'Comments', 'Recommend'])

reviews = []
for col, row in table.iterrows():
  reviews.append(row['Comments'])

table
review_pos_neg = []                                         # stores overall positivity or negativity of reviews
nlp = spacy.load("en_core_web_sm")

# 80/20 split: 80% of data allocated for training,
# 20% for testing the model.
train = []
for col, row in table.iloc[0:2882].iterrows():
  if row['Recommend'] < 3:
    train.append([row['Comments'], "negative"])
  else:
    train.append([row['Comments'], "positive"])

for col, row in table.iloc[1802:3602].iterrows():
  if row['Recommend'] < 3:
    print(row['Comments'])

model = NaiveBayesClassifier(train)
test = reviews[1802:3602]
negative_reviews = 0
for col, row in table.iloc[1802:3602].iterrows():
  if row['Recommend'] < 3:
    print(row['Comments'])
    negative_reviews += 1

for review in test:
  review_aspects = []
  sentences = review.split(". ")
  index = 0
  for sentence in sentences:
    chopped_sentence = nlp(sentence)

    # format for tokens (linking verb, ..., ADV, ADV, ADJ)
    if "was" in sentence or "is" in sentence or "were" in sentence or "are" in sentence:
      for token in chopped_sentence:
        token_describer = ""
        if token.pos_ == 'ADJ':
          adverbs = ""
          for adverb in token.children:
            if adverb.pos_ == 'ADV':
              adverbs += adverb.text + " "
          token_describer = adverbs + token.text
      review_aspects.append({'sentence number': index, 'description': token_describer})
      if not token_describer:
        review_aspects.pop()
      index += 1

    # analyze aspects to choose exceptionally polarizing ones
    polar_aspects = []
    for aspect in review_aspects:
      if TextBlob(aspect['description']).sentiment.polarity >= 0.1:
        polar_aspects.append(aspect)

    # classify each sentence with flagged aspects
    analyzed_review = TextBlob(review, classifier=model)
    var = False
    analyzed_sentences = analyzed_review.sentences
    classifications = []                                   # stores whether each review is positive or negative
    for sentence in analyzed_sentences:
      for aspect in polar_aspects:
        if aspect['description'] in sentence:
          var = True
      if var == True:
        classifications.append(sentence.classify())
    if len(classifications) != 0:
      # print(Counter(classifications).most_common(1)[0][0], review)
      review_pos_neg.append([review, Counter(classifications).most_common(1)[0][0]])
# collect dictionary (final_list) from solution 1
raw_reviews = []
reviews_filename = '/content/drive/MyDrive/labeled_data.csv'
with open(reviews_filename, 'r') as reviews_csvfile:
    csvreader = csv.reader(reviews_csvfile)
    next(csvreader)
    for i in range(1000): # 10399
      row = next(csvreader)
      if int(row[3]) != 0:
        review = row[-1]
        review_arr = review.split(":")
        raw_reviews.append(review_arr[-1])
review_words = []
for review in raw_reviews:
  review_arr = review.split(" ")
  for review_word in review_arr:
    if "\"" not in review_word and review_word != "" and not "&" in review_word and review_word != "-" and review_word != "love" and "I" not in review_word and "'" not in review_word and review_word != "got":
      review_words.append(review_word)
stop_words = set(stopwords.words('english'))
with open('/content/drive/MyDrive/common_words.txt','r') as file:
    common_words = file.read()
words_list = [word for word in review_words if not word in stop_words and not word in common_words]
final_list = []
for word in Counter(words_list).most_common(9):
  final_list.append(word[0])

# use word2vec to identify words to related to words in dictionary
# (used pseudo-recursive loop)
for word in final_list:
  related_words = glove_vector.most_similar(word)
  for entry in related_words:
    final_list.append(entry[0])
  if len(final_list) == 100:
    break

for review in review_pos_neg:
  for word in final_list:
    if word in review[0]:
      review[-1] = "negative"
negative_reviews_pred = 0
for row in review_pos_neg:
  if row[-1] == "negative":
    negative_reviews_pred += 1
print("There were " + str(negative_reviews_pred) + " reviews to check.")
print("There were " + str(negative_reviews) + " negative reviews in actuality.")<jupyter_output>There were 25 reviews to check.
There were 102 negative reviews in actuality.