<jupyter_start><jupyter_text>Preprocessing<jupyter_code>import numpy as np
import spacy
import gensim
import os
import re
from gensim import corpora
from nltk.tokenize import sent_tokenize
import pandas as pd
nlp = spacy.load("en_core_web_md")
from bs4 import BeautifulSoup
from os import mkdir
import requests
#Dicitionary used to access full book text in HTML format
urls = {'foucault_madness_and_civ':'https://archive.org/stream/Michel_Foucault_Madness_And_Civilization/Michel%20Foucault%2C%20Richard%20Howard%20%28transl.%29%20-%20Madness%20and%20Civilization_%20A%20History%20of%20Insanity%20in%20the%20Age%20of%20Reason%20%282013%2C%20Vintage%29_djvu.txt',
        'foucault_discipline_and_punish': 'https://archive.org/stream/MichelFoucaultDisciplineAndPunish/Michel%20Foucault%20-%20Discipline%20and%20Punish_djvu.txt',
        'foucault_history_of_sexuality':'https://archive.org/stream/TheHistoryOfSexualityVol13/The-History-Of-Sexuality-Vol-1-3_djvu.txt',
        'chomsky_media_control': 'https://archive.org/stream/media_Noam_Chomsky-Media_Control/Noam_Chomsky-Media_Control_djvu.txt',
        'chomsky_american_power': 'https://archive.org/stream/AmericanPowerAndTheNewMandarins_201805/American%20Power%20And%20The%20New%20Mandarins_djvu.txt',
        'chomsky_manufacturing_consent': 'https://archive.org/stream/revhosatx14/%5BEdward_S._Herman%2C_Noam_Chomsky%5D_Manufacturing_Con%28b-ok.org%29_djvu.txt'}
def file_books(title, link):
    '''Create directories for book from title and link'''
    #Access HTML webpage on Internet Archive
    r = requests.get(link)
    data = r.text
    soup = BeautifulSoup(data)
    #Full text is in <pre> part of HTML doc
    book = soup.pre.string
    #Write book as text file, save file
    with open(f'./data/{title}.txt', 'w', encoding='utf-8') as book_file:
        book_file.write(book)
        #Make a folder for each book
        mkdir(f'./data/{title}_extracts')
def split_book(title, n_lines=5):
    '''Split a text file based on a number of lines, book title'''
    #Find file path based on title
    filepath = f'./data/{title}.txt'
    #Extract directory and filename from file path
    path, filename = os.path.split(filepath)
    #Change path to book's directory
    path += f'/{title}_extracts'
    # filename.split('.') would not work for filenames with more than one .
    basename, ext = os.path.splitext(filename)
    #open input file
    with open(filepath, 'r', encoding='utf-8') as book_file:
        try:
            #open the first output(extract) file
            extract_file = open(os.path.join(path, '{}_{}{}'.format(basename, 0, ext)), 'w', encoding='utf-8')
            #Loop over all lines of input file, number them
            for i, sentence in enumerate(sent_tokenize(book_file.read())):
                #Close extract file and open a new one
                #When the line number % desired n_lines is 0
                if i % n_lines == 0:
                    extract_file.close()
                    #Open the next output file to write the next extract
                    extract_file = open(os.path.join(path, '{}_{}{}'.format(basename, i/100, ext)), 'w', encoding='utf-8')
                #write the line to extract file
                extract_file.write(sentence)
        finally:
            #close last output file
            extract_file.close()
if not os.path.isdir('./data'):  
    os.mkdir('./data')
for title, link in urls.items():
    if not os.path.isdir(f'./data/{title}_extracts'):
        file_books(title, link)
        split_book(title)
def gather_data(path_to_data):
    path = f'./data/{path_to_data}'
    data = {'extracts': [], 'author': []}
    #For file at the given path
    for file in os.listdir(path):
        #If the directory is not a folder
        if os.path.isdir(file) == False:
            #If the file type is .txt
            if file[-3:] == 'txt':
                #Open each text file at the path provided
                with open(os.path.join(path, file), encoding='utf-8') as t:
                    #Read and strip new line signal
                    text = t.read().replace('\n', ' ')
                    data['extracts'].append(str(text))
                    data['author'].append(path_to_data.split('_')[0])
    return data
extracts_dirs = [folder for folder in os.listdir('./data') if (os.path.isdir(f'./data/{folder}') == True)]
df_final = pd.DataFrame({'extracts': [], 'author': []})
for directory in extracts_dirs:
    extracts = gather_data(directory)
    df = pd.DataFrame(extracts, columns = extracts.keys())
    df_final = pd.concat([df_final, df], axis=0)
print(df_final.shape)
df_final = df_final.reset_index()
df_final = df_final[df_final['author'] != '.ipynb']
df_final['author'].value_counts()
df_final.shape
import pickle
file = open(f'./data/dataframe.pkl', 'wb')
pickle.dump(df_final, file)
file.close()<jupyter_output><empty_output><jupyter_text>Model  Train/Test Split<jupyter_code>from sklearn.model_selection import train_test_split
X = df_final['extracts']
y = df_final['author']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, shuffle=True)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, shuffle=True)<jupyter_output><empty_output><jupyter_text>Randomized Search w/ Naive Bayes<jupyter_code>from imblearn.ensemble import BalancedRandomForestClassifier
from imblearn.pipeline import Pipeline
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import RandomUnderSampler
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV
from sklearn.naive_bayes import MultinomialNB
from gensim.parsing.preprocessing import preprocess_string
# from sklearn.pipeline import Pipeline
counter = TfidfVectorizer(
                          stop_words='english',
                          tokenizer=preprocess_string)

rf = RandomForestClassifier()
nb = MultinomialNB()
rusampler = RandomUnderSampler()
pipeline = Pipeline([('counter', counter),
                     ('sampler', rusampler),
                     ('bayes', nb)])
parameters = {
    'counter__max_df': [i/100 for i in range(75, 100)],
    'counter__min_df': range(0, 10),
    'counter__ngram_range': [(1,2), (1, 3), (2,3), (3, 6), (2, 7)],
    'counter__analyzer': ['word', 'char', 'char_wb']
}

rand_search = RandomizedSearchCV(pipeline, parameters, cv=2, n_iter=8, n_jobs=1, verbose=2)
best = rand_search.fit(X_train, y_train)
best.best_estimator_.score(X_val, y_val)
best.best_estimator_.score(X_test, y_test)
best.best_estimator_.predict_proba([''])
import pickle 
filename = './models/nb_model.pkl'
file = open(filename, 'wb')
pickle.dump(best, file)
file.close()
file = open('./models/nb_model.pkl', 'rb')
nb = pickle.load(file)
file.close()<jupyter_output><empty_output><jupyter_text>Word Embeddings with Spacy<jupyter_code>def get_doc_vectors(words):
    # converts a list of words into their word vectors
    return nlp(words).vector
get_doc_vectors('')
vectors = df_final['extracts'].apply(get_doc_vectors).tolist()
from sklearn.decomposition import PCA
import pickle
pca = PCA(2)
word_vecs_2d = pca.fit_transform(vectors)
word_vecs_2d.shape
word_vecs_train = X_train.apply(get_doc_vectors).tolist()
word_vecs_val = X_val.apply(get_doc_vectors).tolist()
word_vecs_test = X_test.apply(get_doc_vectors).tolist()
#Pickle 2D word vectors for use in dash app
file = open(f'./models/word_vectors.pkl', 'wb')
pickle.dump(word_vecs_2d, file)
file.close()
get_doc_vectors('no')
pipeline.steps[1][1].kneighbors()[1]
from imblearn.pipeline import make_pipeline
from sklearn.neighbors import KNeighborsClassifier
for k in range(1,50):
    pca = PCA(2)
    knn = KNeighborsClassifier(algorithm='kd_tree', n_neighbors = k, weights='distance', metric='euclidean')
    pipeline = make_pipeline(pca, rusampler, knn)
    pipeline.fit(word_vecs_train, y_train)
    new_obs_2d = pipeline.named_steps['pca'].transform(get_doc_vectors('').reshape(1, -1))
    print(k, pipeline.score(word_vecs_test, y_test), 
#           pipeline.steps[1][1].kneighbors(new_obs_2d),
          pipeline.predict_proba(get_doc_vectors('').reshape(1, -1)))
pca = PCA(2)
k=2
knn = KNeighborsClassifier(algorithm='kd_tree', n_neighbors = k, weights='distance', metric='euclidean')
pipeline = make_pipeline(pca, knn)
pipeline.fit(word_vecs_train, y_train)
file = open(f'./models/model_k{k}.pkl', 'wb') # 2 is most balanced prediction probabilities
pickle.dump(pipeline, file)
file.close()
pipeline.named_steps
def display_results(text):
    file = open(f'./models/model_k{29}.pkl', 'rb')
    model = pickle.load(file)
    file.close()
    vector = get_doc_vectors(text)
    new_obs = vector.reshape(1, -1)
    print(new_obs.shape)
    new_obs_2d = model.named_steps['pca'].transform(new_obs)
    index = model.named_steps['kneighborsclassifier'].kneighbors(new_obs_2d)[1][0][0]
    text = df_final['extracts'][index]
    return f'The author most likely to have written your sample, "{text}", is {index}.'

display_results('<NAME> was a great author')
plt.scatter(x=word_vecs_2d[:, 0],y=word_vecs_2d[:,1])
import seaborn as sns
import matplotlib.pyplot as plt
plt.figure(figsize=(10, 10))
sns.scatterplot(word_vecs_2d[:,0], word_vecs_2d[:,1], hue=df_final['author'])
plt.title('')<jupyter_output><empty_output><jupyter_text>Word Embeddings w/ Gensim<jupyter_code>from gensim.test.utils import common_texts
from gensim.utils import  simple_preprocess, lemmatize
preprocess_string(df_final['extracts'][3])
import gensim
from gensim import corpora
from gensim.utils import simple_preprocess
def read_corpus(series, tokens_only=False):
    for i, doc in enumerate(series):
        tokens = gensim.utils.simple_preprocess(doc)
        print(tokens)
        if tokens_only:
            yield tokens
        else:
            # For training data, add tags
            yield gensim.models.doc2vec.TaggedDocument(tokens, [i])
next(read_corpus(df_final['extracts']))<jupyter_output>['rl', 'if', 'kjii', 'jp', 'kwsg', 'ffc', 'fffjil', 'ri', 'hscif', 'piini', 'pqb', 'ffi', 'ipil', 'igij', 'ssbhlil', 'ps', 'im', 'iilb', 'ii', 'msjr', 'tashi', 'nst', 'liii', 'sf', 'ic', 'tta', 'ksp', 'mr', 'izlv', 'siirhki', 'hhh', 'fjfi', 'imimmiumnmum', 'mm', 'ar', 'kttaskni', 'ks', 'ssi', 'sr', 'ziubuss', 'fmhlll', 'fiiblii', 'hi', 'hr', 'hhi', 'rwm', 'tret', 'sks', 'nh', 'ii']
StarcoderdataJupyterStructuredCleanDedup