<jupyter_start><jupyter_text>Load preprocessed data If you'd like to play around with this notebook, start by downloading the skipgram dataset from here:https://www.dropbox.com/s/nd1zxh538o6psal/skipgram_full.npzWARNING: it's a 1Gb download, so it may take a while!<jupyter_code>import numpy as np

codes = np.load("../data/skipgram_full.npz")['coded']
# Remove duplicate skipgrams
codes = codes[codes[:, 0] != codes[:, 1]]
code2token = np.load("../data/skipgram_full.npz")['c2t'].tolist()
token2code = np.load("../data/skipgram_full.npz")['t2c'].tolist()
# First column is the first token code
# second column is the 2nd token code
# third column is the skip gram count
# fourth is PMI * 1e6
codes
train_x = codes[:, :2].copy().astype(np.int64)
train_y = codes[:, 3].astype(np.float32) / 1e6
train_y
train_y.max()
top_codes = np.argsort(train_y)[-10:]
[[code2token[c[0]], code2token[c[1]]] for c in codes[top_codes, :2]]
n_user = np.max(train_x[:, :2]) + 1
n_item = np.max(train_x[:, :2]) + 1
n_user<jupyter_output><empty_output><jupyter_text>Define the MF Model<jupyter_code>import torch
from torch import nn
import torch.nn.functional as F

def l2_regularize(array):
    loss = torch.sum(array ** 2.0)
    return loss


class MF(nn.Module):
    itr = 0
    
    def __init__(self, n_user, n_item, k=18, c_vector=1.0, c_bias=1.0, writer=None):
        super(MF, self).__init__()
        self.writer = writer
        self.k = k
        self.n_user = n_user
        self.n_item = n_item
        self.c_bias = c_bias
        self.c_vector = c_vector
        self.user = nn.Embedding(n_user, k)
        self.item = nn.Embedding(n_item, k)
        self.user.weight.data.normal_(0, 1.0 / n_user)
        self.item.weight.data.normal_(0, 1.0 / n_item)
        
        # We've added new terms here:
        self.bias_user = nn.Embedding(n_user, 1)
        self.bias_item = nn.Embedding(n_item, 1)
        self.bias = nn.Parameter(torch.ones(1))

    
    def __call__(self, train_x):
        user_id = train_x[:, 0]
        item_id = train_x[:, 1]
        vector_user = self.user(user_id)
        vector_item = self.item(item_id)
        bias_user = self.bias_user(user_id).squeeze()
        bias_item = self.bias_item(item_id).squeeze()
        biases = (self.bias + bias_user + bias_item)
        ui_interaction = torch.sum(vector_user * vector_item, dim=1)
        prediction = ui_interaction + biases
        return prediction
    
    def loss(self, prediction, target):
        loss_mse = F.mse_loss(prediction, target.squeeze())
        prior_bias_user =  l2_regularize(self.bias_user.weight) * self.c_bias
        prior_bias_item = l2_regularize(self.bias_item.weight) * self.c_bias
        prior_user =  l2_regularize(self.user.weight) * self.c_vector
        prior_item = l2_regularize(self.item.weight) * self.c_vector
        total = loss_mse #+ prior_user + prior_item
        for name, var in locals().items():
            if type(var) is torch.Tensor and var.nelement() == 1 and self.writer is not None:
                self.writer.add_scalar(name, var, self.itr)
        return total<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>from ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator
from ignite.metrics import Loss
from tensorboardX import SummaryWriter
from ignite.metrics import MeanSquaredError

from loader import Loader
from datetime import datetime<jupyter_output><empty_output><jupyter_text>Hyperparameters<jupyter_code>lr = 1e-3
k = 128
c_bias = 1e-9
c_vector = 1e-9
log_dir = 'runs/simple_mf_05_word2vec_' + str(datetime.now()).replace(' ', '_')
print(log_dir)
writer = SummaryWriter(log_dir=log_dir)
model = MF(n_user, n_item,  k=k, c_bias=c_bias, 
           c_vector=c_vector, writer=writer)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
trainer = create_supervised_trainer(model, optimizer, model.loss)
metrics = {'accuracy': MeanSquaredError()}
train_loader = Loader(train_x, train_y, batchsize=1024)


def log_training_loss(engine, log_interval=400):
    epoch = engine.state.epoch
    itr = engine.state.iteration
    fmt = "Epoch[{}] Iteration[{}/{}] Loss: {:.2f}"
    msg = fmt.format(epoch, itr, len(train_loader), engine.state.output)
    model.itr = itr
    if itr % log_interval == 0:
        print(msg)

trainer.add_event_handler(event_name=Events.ITERATION_COMPLETED, handler=log_training_loss)

model
model.load_state_dict(torch.load("model_05_word2vec"))<jupyter_output><empty_output><jupyter_text>Run model<jupyter_code>trainer.run(train_loader, max_epochs=25)
torch.save(model.state_dict(), "model_05_word2vec")<jupyter_output><empty_output><jupyter_text>Save the embeddings<jupyter_code>label_token = ['|' + code2token[c] for c in range(n_user)]
writer.add_embedding(model.user.weight)
# writer.add_embedding(model.item.weight, metadata=label_token)<jupyter_output><empty_output><jupyter_text>Introspect the model Evaluate what urban dictionary thinks are similar words.<jupyter_code>vectors_raw = model.user.weight.data.numpy()
vectors = vectors_raw / np.sqrt((vectors_raw**2.0).sum(axis=1)[:, None])
(vectors[0]**2.0).sum()
def find_closest(token, n=10):
    code = token2code[token]
    vector = vectors[code]
    similarity = np.sum(vector[None, :] * vectors, axis=1)
    closest = np.argsort(similarity)[::-1]
    for code in closest[1:n]:
        print(code2token[code], similarity[code])
find_closest('dude')
find_closest('netflix')
find_closest('lol')
find_closest('hipster')
find_closest('crunk')
find_closest('bromance')
find_closest('barbie')
find_closest('relationship')
find_closest('pope')
find_closest('trump')
find_closest('selfie')<jupyter_output>selfies 0.6768813
instagram 0.58078086
photo 0.5547765
pic 0.5447346
snapchat 0.54272944
upload 0.52603865
photographer 0.5154379
caption 0.49573278
tweet 0.47855204<jupyter_text>Subtract and add word vectors<jupyter_code>def add_subtract(center, minus, plus, n=10):
    vector = (vectors[token2code[center]]
             - vectors[token2code[minus]]
             + vectors[token2code[plus]])
    similarity = np.sum(vector[None, :] * vectors, axis=1)
    closest = np.argsort(similarity)[::-1]
    for code in closest[2:n]:
        print(code2token[code])
add_subtract('burrito', 'mexican', 'italian')
add_subtract('drunk', 'beer', 'weed')<jupyter_output>drunk
ganja
shrooms
chronic
paranoid
pothead
kush
fucked