<jupyter_start><jupyter_text>Testing embedding modelsIn this notebook we are going to setup our LSTM model for the word generation task.<jupyter_code>from __future__ import print_function

import numpy as np
import gensim
import string

from keras.callbacks import LambdaCallback
from keras.layers.recurrent import LSTM
from keras.layers.embeddings import Embedding
from keras.layers import Dense, Activation
from keras.models import Sequential
from keras.utils.data_utils import get_file<jupyter_output>Using TensorFlow backend.<jupyter_text>Load models<jupyter_code>word_model = gensim.models.Word2Vec.load('./data/meta-n2v/meta-n2v100MB')
pretrained_weights = word_model.wv.syn0
vocab_size, emdedding_size = pretrained_weights.shape
print('Result embedding shape:', pretrained_weights.shape)
print('Checking similar words:')

for word in ['model']:
  most_similar = ', '.join('%s (%.2f)' % (similar, dist) for similar, dist in word_model.most_similar(word)[:8])
  print('  %s -> %s' % (word, most_similar))
# Now for loading the corresponding amount of sentences we need to pass the number of sentences parameter
from sentence_loader import lazy_load
tokenised_sents, sents = lazy_load(chunk_size=10240)
print(len(sents))
#ub = np.max([len(sent) for sent in tokenised_sents])

long_sents = 0
for sent in tokenised_sents:
    if len(sent) > 45:
        long_sents += 1

long_sents
# Let this length be the upperbound. 
# reformat the sentences to have length no more than the maximum length
sentences = [sentence for sentence in tokenised_sents if len(sentence) < 45]
print(len(sentences))
print(sentences[0])
max_sentence_len = 45

def word2idx(word):
  return word_model.wv.vocab[word].index
def idx2word(idx):
  return word_model.wv.index2word[idx]
train_x = np.zeros([len(sentences), max_sentence_len], dtype=np.int32)
train_y = np.zeros([len(sentences)], dtype=np.int32)
for i, sentence in enumerate(sentences):
  for t, word in enumerate(sentence[:-1]):
    train_x[i, t] = word2idx(word)
  train_y[i] = word2idx(sentence[-1])
print('train_x shape:', train_x.shape)
print('train_y shape:', train_y.shape)<jupyter_output>train_x shape: (70, 45)
train_y shape: (70,)<jupyter_text>Training<jupyter_code>model = Sequential()
model.add(Embedding(input_dim=vocab_size, output_dim=emdedding_size, weights=[pretrained_weights]))
model.add(LSTM(units=emdedding_size))
model.add(Dense(units=vocab_size))
model.add(Activation('softmax'))
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')
# implement a random sampling mechanism

def sample(preds, temperature=1.0):
  if temperature <= 0:
    return np.argmax(preds)
  preds = np.asarray(preds).astype('float64')
  preds = np.log(preds) / temperature
  exp_preds = np.exp(preds)
  preds = exp_preds / np.sum(exp_preds)
  probas = np.random.multinomial(1, preds, 1)
  return np.argmax(probas)

def generate_next(text, num_generated=10):
  word_idxs = [word2idx(word) for word in text.lower().split()]
  for i in range(num_generated):
    prediction = model.predict(x=np.array(word_idxs))
    idx = sample(prediction[-1], temperature=0.7)
    word_idxs.append(idx)
  return ' '.join(idx2word(idx) for idx in word_idxs)
def on_epoch_end(epoch, _):
  print('\nGenerating text after epoch: %d' % epoch)
  texts = [
    'chess',
    'is',
    'fantasy',
    'by',
    'steve'
  ]
  for text in texts:
    sample = generate_next(text)
    print('%s... -> %s' % (text, sample))
model.fit(train_x, train_y,
          batch_size=128,
          epochs=20,
          callbacks=[LambdaCallback(on_epoch_end=on_epoch_end)])
%%file lstm_model.py

from __future__ import print_function
import numpy as np

from keras.callbacks import LambdaCallback
from keras.layers.recurrent import LSTM
from keras.layers.embeddings import Embedding
from keras.layers import Dense, Activation
from keras.models import Sequential

def sample(preds, temperature=1.0):
    if temperature <= 0:
        return np.argmax(preds)
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

def eval_on_lstm(tokenised_sents, word_model, max_sentence_len, test_ratio=0.2):
    
    sentences = [sentence for sentence in tokenised_sents if len(sentence) < max_sentence_len]
    pretrained_weights = word_model.wv.syn0
    vocab_size, emdedding_size = pretrained_weights.shape
    def word2idx(word):
        try:
            idx = word_model.wv.vocab[word].index
        except:
            print("word: {} not in vocab using default word card\n".format(word))
            idx = 0
        return idx
    
    def idx2word(idx):
        return word_model.wv.index2word[idx]
    
    total = len(sentences)
    train_size = int(total * (1 - test_ratio))
    test_size = total - train_size
    
    train_x = np.zeros([train_size, max_sentence_len], dtype=np.int32)
    train_y = np.zeros([train_size], dtype=np.int32)
    test_x = np.zeros([test_size, max_sentence_len], dtype=np.int32)
    test_y = np.zeros([test_size], dtype=np.int32)

    for i, sentence in enumerate(sentences[:train_size]):
        for t, word in enumerate(sentence[:-1]):
            train_x[i, t] = word2idx(word)
        train_y[i] = word2idx(sentence[-1])
    for i, sentence in enumerate(sentences[train_size:]):
        for t, word in enumerate(sentence[:-1]):
            test_x[i, t] = word2idx(word)
        test_y[i] = word2idx(sentence[-1])
    
    model = Sequential()
    model.add(Embedding(input_dim=vocab_size, output_dim=emdedding_size, weights=[pretrained_weights]))
    model.add(LSTM(units=emdedding_size))
    model.add(Dense(units=vocab_size))
    model.add(Activation('softmax'))
    model.compile(optimizer='adam', loss='sparse_categorical_crossentropy')
    
    def generate_next(text, num_generated=10):
        word_idxs = [word2idx(word) for word in text.lower().split()]
        for i in range(num_generated):
            prediction = model.predict(x=np.array(word_idxs))
            idx = sample(prediction[-1], temperature=0.7)
            word_idxs.append(idx)
        return ' '.join(idx2word(idx) for idx in word_idxs)
    
    def on_epoch_end(epoch, _):
        print('\nGenerating text after epoch: %d' % epoch)
        texts = [
        'chess',
        'is',
        'fantasy',
        'by',
        'steve',
        'lasting'
        ]
        for text in texts:
            sample = generate_next(text)
            print('%s... -> %s' % (text, sample))
            
    model.fit(train_x, train_y,
          batch_size=128,
          epochs=20,
          callbacks=[LambdaCallback(on_epoch_end=on_epoch_end)])
    
    scores = model.evaluate(test_x, test_y, verbose=0)
    print("Accuracy of the model is {}".format(scores))
from lstm_model import eval_on_lstm
eval_on_lstm(tokenised_sents,word_model, 40)
# now lets test the 1Mb model of syntactic_n2v
from sentence_loader import lazy_load
tokenised_sents, sents = lazy_load(chunk_size=1048576/2)
len(sents)
from lstm_model import eval_on_lstm
import gensim
word_model = gensim.models.Word2Vec.load('./data/syncode/syncode_model_1MB')
eval_on_lstm(tokenised_sents,word_model, 40)
word_model = gensim.models.Word2Vec.load('./data/w2v/w2v_model_1MB')
eval_on_lstm(tokenised_sents,word_model, 40)
%%file evaluate_models.py

from sentence_loader import lazy_load
from lstm_model import eval_on_lstm
import gensim

sizes = [1,2,4,8]
tokenized_sents = []
sents = []
for size in sizes:
    ts, s = lazy_load(chunk_size=(10240)*size)
    tokenized_sents.extend(ts)
    sents.extend(s)
    word_model = gensim.models.Word2Vec.load('./data/syncode/syncode_model_'+str(size)+'MB')
    eval_on_lstm(tokenized_sents,word_model, 40)
    print "\n done with syncode "+ str(size)+"\n"
    word_model = gensim.models.Word2Vec.load('./data/w2v/w2v_model_'+str(size)+'MB')
    eval_on_lstm(tokenized_sents,word_model, 40)
    print "\n done with word2vec "+ str(size)+"\n"<jupyter_output>Overwriting evaluate_models.py