# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: slack-nlp-Xp963HYr
#     language: python
#     name: slack-nlp-xp963hyr
# ---

# +
import sys
import pandas as pd

from gensim.models import FastText, Word2Vec, Doc2Vec
from gensim.test.utils import get_tmpfile

src_dir = "../src/"
sys.path.append(src_dir)
from data.processing import format_ts, tokenise_text_col

# -

df = pd.read_csv("../data/processed/clean_tokenised_text.csv", index_col = 0)

# tokenise text

df = tokenise_text_col(df, text_col = "text")

# we ignore blanks

mask = [not("nan" in text) for text in df.text_tokens]

# we get the tokenised messages and build a FastText model

messages = df[mask].text_tokens.tolist()

# try a fasttext model

ft_mdl = FastText(messages, size = 100, window = 5, min_count = 3, iter = 10)

ft_mdl.wv.most_similar("roadburn")

# ok, lets try word2vec

w2v_mdl = Word2Vec(
        messages,
        size=150,
        window=10,
        min_count=2,
        workers=10)

w2v_mdl.wv.most_similar("roadburn")

# ok, save these models



StarcoderdataJupyterScriptsDedupFiltered