Refer to :func:`word_embedding_loader.loader.glove.load_with_vocab` for the API.
763261
def load_with_vocab(fin, vocab, dtype=np.float32):
    
    line = next(fin)
    data = line.strip().split(b' ')
    assert len(data) == 2
    size = int(data[1])
    arr = np.empty((len(vocab), size), dtype=dtype)
    arr.fill(np.NaN)
    for line in fin:
        token, v = _load_line(line, dtype, size)
        if token in vocab:
            arr[vocab[token], :] = v
    if np.any(np.isnan(arr)):
        raise ParseError(b"Some of vocab was not found in word embedding file")
    return arr
1054398
def load_word_file(filename):
    """"""
    words_file = resource_filename(__name__, "words/%s" % filename)
    handle = open(words_file, 'r')
    words = handle.readlines()
    handle.close()
    return words
1330420
def _init_vocab_from_list(self, vocab_list):
    
    def token_gen():
      for token in vocab_list:
        if token not in RESERVED_TOKENS:
          yield token

    self._init_vocab(token_gen())
58401
def _build_vocab(filename, vocab_dir, vocab_name):
  
  vocab_path = os.path.join(vocab_dir, vocab_name)
  if not tf.gfile.Exists(vocab_path):
    with tf.gfile.GFile(filename, "r") as f:
      data = f.read().split()
    counter = collections.Counter(data)
    count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
    words, _ = list(zip(*count_pairs))
    encoder = text_encoder.TokenTextEncoder(None, vocab_list=words)
    encoder.store_to_file(vocab_path)
  else:
    encoder = text_encoder.TokenTextEncoder(vocab_path)
  return encoder
321676
def get_word_level_vocab(self):
        

        def simple_tokenize(source_str, token_delim=' ', seq_delim='\n'):
            return list(filter(None, re.split(token_delim + '|' + seq_delim, source_str)))

        return VocabProvider._create_squad_vocab(simple_tokenize, self._dataset)
403406
def _get_or_generate_vocab(tmp_dir, vocab_filename, vocab_size):
  """"""
  vocab_filepath = os.path.join(tmp_dir, vocab_filename)
  print('Vocab file written to: ' + vocab_filepath)

  if tf.gfile.Exists(vocab_filepath):
    gs = text_encoder.SubwordTextEncoder(vocab_filepath)
    return gs
  example_file = os.path.join(tmp_dir, _EXAMPLES_FILE)
  gs = text_encoder.SubwordTextEncoder()
  token_counts = tokenizer.corpus_token_counts(
      example_file, corpus_max_lines=1000000)
  gs = gs.build_to_target_size(
      vocab_size, token_counts, min_val=1, max_val=1e3)
  gs.store_to_file(vocab_filepath)
  return gs
764853
def get_or_generate_txt_vocab(data_dir, vocab_filename, vocab_size,
                              filepatterns):
  """"""
  if isinstance(filepatterns, str):
    filepatterns = [filepatterns]

  def generate():
    tf.logging.info("Generating vocab from %s", filepatterns)
    for filepattern in filepatterns:
      for filename in tf.gfile.Glob(filepattern):
        with tf.gfile.GFile(filename, mode="r") as source_file:
          for line in source_file:
            yield line.strip()

  return get_or_generate_vocab_inner(data_dir, vocab_filename, vocab_size,
                                     generate())
701538
def build_vocab(dataset):
    
    counter = nlp.data.count_tokens([w for e in dataset for s in e[:2] for w in s],
                                    to_lower=True)
    vocab = nlp.Vocab(counter)
    return vocab
7310
def from_json(cls, json_str):
        
        vocab_dict = json.loads(json_str)

        unknown_token = vocab_dict.get('unknown_token')
        vocab = cls(unknown_token=unknown_token)
        vocab._idx_to_token = vocab_dict.get('idx_to_token')
        vocab._token_to_idx = vocab_dict.get('token_to_idx')
        if unknown_token:
            vocab._token_to_idx = DefaultLookupDict(vocab._token_to_idx[unknown_token],
                                                    vocab._token_to_idx)
        vocab._reserved_tokens = vocab_dict.get('reserved_tokens')
        vocab._padding_token = vocab_dict.get('padding_token')
        vocab._bos_token = vocab_dict.get('bos_token')
        vocab._eos_token = vocab_dict.get('eos_token')
        return vocab
542077
def _get_embed(self, embed, vocab_size, embed_size, initializer, dropout, prefix):
        
        if embed is None:
            assert embed_size is not None, '"embed_size" cannot be None if "word_embed" or ' \
                                           'token_type_embed is not given.'
            with self.name_scope():
                embed = nn.HybridSequential(prefix=prefix)
                with embed.name_scope():
                    embed.add(nn.Embedding(input_dim=vocab_size, output_dim=embed_size,
                                           weight_initializer=initializer))
                    if dropout:
                        embed.add(nn.Dropout(rate=dropout))
        assert isinstance(embed, Block)
        return embed
843092
def load_pretrained_word2vec(infile):
    """"""
    if isinstance(infile, str):
        infile = open(infile)

    word2vec_list = {}
    for idx, line in enumerate(infile):
        if idx == 0:
            vocab_size, dim = line.strip().split()
        else:
            tks = line.strip().split()
            word2vec_list[tks[0]] = map(float, tks[1:])

    return word2vec_list
876331
def _get_token_encoder(vocab_dir, vocab_name, filename):
  """"""
  vocab_path = os.path.join(vocab_dir, vocab_name)
  if not tf.gfile.Exists(vocab_path):
    _build_vocab(filename, vocab_path, 10000)
  return text_encoder.TokenTextEncoder(vocab_path)
913327
def collect_vocab(qp_pairs):
    '''
    
    '''
    vocab = set()
    for qp_pair in qp_pairs:
        for word in qp_pair['question_tokens']:
            vocab.add(word['word'])
        for word in qp_pair['passage_tokens']:
            vocab.add(word['word'])
    return vocab
34254
def load_glove(file):
    
    model = {}
    with open(file, encoding="utf8", errors='ignore') as f:
        for line in f:
            line = line.split(' ')
            word = line[0]
            vector = np.array([float(val) for val in line[1:]])
            model[word] = vector

    return model
1281503
def set_word_at_rva(self, rva, word):
        """"""
        return self.set_bytes_at_rva(rva, self.get_data_from_word(word))
1111121
def get_or_create_vocab(self, data_dir, tmp_dir, force_get=False):
    """"""
    # We assume that vocab file is present in data_dir directory where the
    # data generated will be stored.
    vocab_filepath = os.path.join(data_dir, self.vocab_filename)
    encoder = text_encoder.SubwordTextEncoder(vocab_filepath)
    return encoder
1074695
def get_label_vocab(vocab_path):
  """"""
  if vocab_path:
    try:
      with tf.io.gfile.GFile(vocab_path, 'r') as f:
        return [line.rstrip('\n') for line in f]
    except tf.errors.NotFoundError as err:
      tf.logging.error('error reading vocab file: %s', err)
  return []