# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

import zipfile
zip_ref = zipfile.ZipFile('Corpuses.zip', 'r')
zip_ref.extractall('')
zip_ref.close()

import codecs

one_long_string = ""
with codecs.open('Corpuses/MarkTwain.txt', 'r', 'utf-8-sig') as text_file:
    one_long_string = text_file.read()

one_long_string[99000:99900]

from nltk import word_tokenize, sent_tokenize

sentences = sent_tokenize(one_long_string)
del(one_long_string)

sentences[200:205]

tokenized_sentences = map(word_tokenize, sentences)
del(sentences)

print(tokenized_sentences[200:205])

from nltk import download

download('stopwords')

from nltk.stem import WordNetLemmatizer

wordnet_lemmatizer = WordNetLemmatizer()

lemmatized_sentences = map(lambda sentence: map(wordnet_lemmatizer.lemmatize, sentence), tokenized_sentences)

print(lemmatized_sentences[200:205])

del(tokenized_sentences)

from nltk import download

download('averaged_perceptron_tagger')

from nltk import pos_tag, pos_tag_sents

pos_tag(word_tokenize('Cats, cat, Cat, and "The Cats"'))

pos_sentences = pos_tag_sents(lemmatized_sentences)
del(lemmatized_sentences)

print(pos_sentences[200:205])

download('tagsets')

from nltk.help import upenn_tagset

upenn_tagset()

# https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html

# Note the ``(u'Paris', 'NNP'), (u'Exposition', 'NNP'), (u'Americans', 'NNPS')``
#
# ``NNP`` stands for proper noun
#
# ``NNPS``	proper noun, plural
#
# We need to get rid of all capital letters in not-proper nouns and from all punctuation marks and numbers.

# tags_to_delete = ['$', "''", "(", ")", ",", "--", ".", ":", "CC"]
tags_to_not_lowercase = set(['NNP', 'NNPS'])
tags_to_preserve = set(['JJ', 'JJR', 'JJS', 'NN', 'NNP', 'NNPS', 'NNS', 'RB', 'RBR', 'RBS','UH', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ'])

print(pos_sentences[203])


def carefully_lowercase(words):
    return [(word.lower(), pos) if pos not in tags_to_not_lowercase else (word, pos)
            for (word, pos) in words]


def filter_meaningful(words):
    return [word for (word, pos) in words if pos in tags_to_preserve]


res = map(carefully_lowercase, pos_sentences[203:205])
print(res)

filtered = map(filter_meaningful, res)
del(res)
print(filtered)

lowercased_pos_sentences = map(carefully_lowercase,  pos_sentences)
del(pos_sentences)

sentences_to_train_on = map(lambda words: [word for (word, pos) in words], lowercased_pos_sentences)

print(sentences_to_train_on[203:205])

import itertools

filtered = map(filter_meaningful, lowercased_pos_sentences)
flatten = list(itertools.chain(*filtered))
words_to_keep = set(flatten)

del(filtered, flatten, lowercased_pos_sentences)

from nltk.corpus import stopwords
import string

stop_words = set(stopwords.words('english') + list(string.punctuation) + ['wa'])

import gensim


def trim_rule(word, count, min_count):
    if word not in words_to_keep or word in stop_words:
        return gensim.utils.RULE_DISCARD
    else:
        return gensim.utils.RULE_DEFAULT


model = gensim.models.Word2Vec(sentences_to_train_on, min_count=15, trim_rule=trim_rule)

model.most_similar('house', topn=5)

model.most_similar('America', topn=5)

model.most_similar('water', topn=5)

model.most_similar('money', topn=5)

model.most_similar('cat', topn=5)

model.wv.save_word2vec_format(fname='MarkTwain.bin', binary=True)



StarcoderdataJupyterScriptsDedupFiltered