<jupyter_start><jupyter_text>Tokenize<jupyter_code>def tokenize(doc_df, remove_pos_tuple=False, OHCO=OHCO):
    
    # Paragraphs to Sentences
    df = doc_df.para_str\
        .apply(lambda x: pd.Series(nltk.sent_tokenize(x)))\
        .stack()\
        .to_frame()\
        .rename(columns={0:'sent_str'})
    
    # Sentences to Tokens
    # .apply(lambda x: pd.Series(nltk.pos_tag(nltk.word_tokenize(x))))\
    df = df.sent_str\
        .apply(lambda x: pd.Series(nltk.pos_tag(nltk.WhitespaceTokenizer().tokenize(x))))\
        .stack()\
        .to_frame()\
        .rename(columns={0:'pos_tuple'})
    
    # Grab info from tuple
    df['pos'] = df.pos_tuple.apply(lambda x: x[1])
    df['token_str'] = df.pos_tuple.apply(lambda x: x[0])
    if remove_pos_tuple:
        df = df.drop('pos_tuple', 1)
    
    # Add index
    df.index.names = OHCO
    
    return df
TOKEN = tokenize(DOC)
TOKEN.sample(25)
TOKEN['term_str'] = TOKEN['token_str'].str.lower().str.replace('[\W_]', '')

VOCAB = TOKEN.term_str.value_counts().to_frame().rename(columns={'index':'term_str', 'term_str':'n'})\
    .sort_index().reset_index().rename(columns={'index':'term_str'})
VOCAB.index.name = 'term_id'

VOCAB['num'] = VOCAB.term_str.str.match("\d+").astype('int')
VOCAB
sw = pd.DataFrame(nltk.corpus.stopwords.words('english'), columns=['term_str'])
sw = sw.reset_index().set_index('term_str')
sw.columns = ['dummy']
sw.dummy = 1
sw.sample(10)
VOCAB['stop'] = VOCAB.term_str.map(sw.dummy)
VOCAB['stop'] = VOCAB['stop'].fillna(0).astype('int')
VOCAB[VOCAB.stop == 1].sample(10)<jupyter_output><empty_output><jupyter_text>Porter Stems<jupyter_code>from nltk.stem.porter import PorterStemmer

stemmer = PorterStemmer()
VOCAB['p_stem'] = VOCAB.term_str.apply(stemmer.stem)

VOCAB.sample(10)
TOKEN
token1 = TOKEN
pos_max = token1.groupby(['term_str',"pos"]).count().sort_values("token_str", ascending = False).groupby(level=0).head(1)\
    .reset_index().set_index('term_str')
pos_max.sort_index().tail(200)
VOCAB['pos_max'] = VOCAB.term_str.map(pos_max.pos)
VOCAB<jupyter_output><empty_output><jupyter_text>save csvs<jupyter_code>DOC.to_csv('DOC.csv')
LIB.to_csv('LIB.csv')
VOCAB.to_csv('VOCAB.csv')
TOKEN.to_csv('TOKEN.csv')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup