Read pre-trained embedding file for extending vocabulary

        Parameters
        ----------
        pret_embeddings : tuple
            (embedding_name, source), used for gluonnlp.embedding.create(embedding_name, source)
776243
def _add_pret_words(self, pret_embeddings):
        
        words_in_train_data = set(self._id2word)
        pret_embeddings = gluonnlp.embedding.create(pret_embeddings[0], source=pret_embeddings[1])

        for idx, token in enumerate(pret_embeddings.idx_to_token):
            if token not in words_in_train_data:
                self._id2word.append(token)
1030532
def _load(cls,
              config: Params,
              serialization_dir: str,
              weights_file: str = None,
              cuda_device: int = -1) -> 'Model':
        
        model_params = config.get('model')

        # The experiment config tells us how to _train_ a model, including where to get pre-trained
        # embeddings from.  We're now _loading_ the model, so those embeddings will already be
        # stored in our weights.  We don't need any pretrained weight file anymore, and we don't
        # want the code to look for it, so we remove it from the parameters here.
        remove_pretrained_embedding_params(model_params)
        model = Model.from_params(vocab=None, params=model_params)

        # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are
        # in sync with the weights
        if cuda_device >= 0:
            model.cuda(cuda_device)
        else:
            model.cpu()

        return model
802372
def load_pretrained(self, wgts_fname:str, itos_fname:str, strict:bool=True):
        ""
        old_itos = pickle.load(open(itos_fname, 'rb'))
        old_stoi = {v:k for k,v in enumerate(old_itos)}
        wgts = torch.load(wgts_fname, map_location=lambda storage, loc: storage)
        if 'model' in wgts: wgts = wgts['model']
        wgts = convert_weights(wgts, old_stoi, self.data.train_ds.vocab.itos)
        self.model.load_state_dict(wgts, strict=strict)
291587
def extend_from_instances(self,
                              params: Params,
                              instances: Iterable['adi.Instance'] = ()) -> None:
        
        min_count = params.pop("min_count", None)
        max_vocab_size = pop_max_vocab_size(params)
        non_padded_namespaces = params.pop("non_padded_namespaces", DEFAULT_NON_PADDED_NAMESPACES)
        pretrained_files = params.pop("pretrained_files", {})
        min_pretrained_embeddings = params.pop("min_pretrained_embeddings", None)
        only_include_pretrained_words = params.pop_bool("only_include_pretrained_words", False)
        tokens_to_add = params.pop("tokens_to_add", None)
        params.assert_empty("Vocabulary - from dataset")

        logger.info("Fitting token dictionary from dataset.")
        namespace_token_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
        for instance in Tqdm.tqdm(instances):
            instance.count_vocab_items(namespace_token_counts)
        self._extend(counter=namespace_token_counts,
                     min_count=min_count,
                     max_vocab_size=max_vocab_size,
                     non_padded_namespaces=non_padded_namespaces,
                     pretrained_files=pretrained_files,
                     only_include_pretrained_words=only_include_pretrained_words,
                     tokens_to_add=tokens_to_add,
                     min_pretrained_embeddings=min_pretrained_embeddings)
958276
def _build_graph(self, vocab, initial_embedding_dict):
        
        # Constants
        self.ones = tf.ones([self.n_words, 1])

        # Parameters:
        if initial_embedding_dict is None:
            # Ordinary GloVe
            self.W = self._weight_init(self.n_words, self.n, 'W')
            self.C = self._weight_init(self.n_words, self.n, 'C')
        else:
            # This is the case where we have values to use as a
            # "warm start":
            self.n = len(next(iter(initial_embedding_dict.values())))
            W = randmatrix(len(vocab), self.n)
            C = randmatrix(len(vocab), self.n)
            self.original_embedding = np.zeros((len(vocab), self.n))
            self.has_embedding = np.zeros(len(vocab))
            for i, w in enumerate(vocab):
                if w in initial_embedding_dict:
                    self.has_embedding[i] = 1.0
                    embedding = np.array(initial_embedding_dict[w])
                    self.original_embedding[i] = embedding
                    # Divide the original embedding into W and C,
                    # plus some noise to break the symmetry that would
                    # otherwise cause both gradient updates to be
                    # identical.
                    W[i] = 0.5 * embedding + noise(self.n)
                    C[i] = 0.5 * embedding + noise(self.n)
            self.W = tf.Variable(W, name='W', dtype=tf.float32)
            self.C = tf.Variable(C, name='C', dtype=tf.float32)
            self.original_embedding = tf.constant(self.original_embedding,
                                                  dtype=tf.float32)
            self.has_embedding = tf.constant(self.has_embedding,
                                             dtype=tf.float32)
            # This is for testing. It differs from
            # `self.original_embedding` only in that it includes the
            # random noise we added above to break the symmetry.
            self.G_start = W + C

        self.bw = self._weight_init(self.n_words, 1, 'bw')
        self.bc = self._weight_init(self.n_words, 1, 'bc')

        self.model = tf.tensordot(self.W, tf.transpose(self.C), axes=1) + \
                     tf.tensordot(self.bw, tf.transpose(self.ones), axes=1) + \
                     tf.tensordot(self.ones, tf.transpose(self.bc), axes=1)
331461
def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split):
    
    generator = self.generate_samples(data_dir, tmp_dir, dataset_split)
    encoder = self.get_or_create_vocab(data_dir, tmp_dir)
    label_encoder = self.get_labels_encoder(data_dir)
    for sample in generator:
      inputs = encoder.encode(sample["inputs"])
      inputs.append(text_encoder.EOS_ID)
      context = encoder.encode(sample["context"])
      context.append(text_encoder.EOS_ID)
      targets = label_encoder.encode(sample["targets"])
      sample["targets"] = targets
      yield {"inputs": inputs, "context": context, "targets": targets}
755475
def main():
    
    setup_main_logger(console=True, file_logging=False)
    params = argparse.ArgumentParser(description='Quick usage: python3 -m sockeye.init_embedding '
                                                 '-w embed-in-src.npy embed-in-tgt.npy '
                                                 '-i vocab-in-src.json vocab-in-tgt.json '
                                                 '-o vocab-out-src.json vocab-out-tgt.json '
                                                 '-n source_embed_weight target_embed_weight '
                                                 '-f params.init')
    arguments.add_init_embedding_args(params)
    args = params.parse_args()
    init_embeddings(args)
364078
def dataset_iterator(self, file_path: str) -> Iterator[OntonotesSentence]:
        
        for conll_file in self.dataset_path_iterator(file_path):
            yield from self.sentence_iterator(conll_file)
843092
def load_pretrained_word2vec(infile):
    """"""
    if isinstance(infile, str):
        infile = open(infile)

    word2vec_list = {}
    for idx, line in enumerate(infile):
        if idx == 0:
            vocab_size, dim = line.strip().split()
        else:
            tks = line.strip().split()
            word2vec_list[tks[0]] = map(float, tks[1:])

    return word2vec_list
1035823
def _example_generator(filename):
  
  for idx, line in enumerate(tf.gfile.Open(filename, "rb")):
    if idx == 0: continue  # skip header
    line = text_encoder.to_unicode_utf8(line.strip())
    split_line = line.split("\t")
    # Works for both splits even though dev has some extra human labels.
    yield {
        "premise": split_line[8],
        "hypothesis": split_line[9],
        "label": split_line[-1]
    }
1318205
def read_paraphrase_file(filename):
    '''
    
    '''
    with open(filename, 'rb') as input_file:
        doc = etree.parse(input_file)

    assert doc.getroot().tag == 'wiktionaryParaphrases'
    paraphrases = []
    for child in doc.getroot():
        if child.tag == 'wiktionaryParaphrase':
            paraphrase = child
            warn_attribs('', paraphrase, PARAPHRASE_ATTRIBS)
            if 0 < len(paraphrase):
                print('unrecognised child of <wiktionaryParaphrase>',
                      list(paraphrase))
            paraphrase_dict = dict(paraphrase.items())
            if paraphrase_dict['edited'] not in MAP_YESNO_TO_BOOL:
                print('<paraphrase> attribute "edited" has unexpected value',
                      paraphrase_dict['edited'])
            else:
                paraphrase_dict['edited'] = MAP_YESNO_TO_BOOL[
                    paraphrase_dict['edited']]
            if not paraphrase_dict['wiktionarySenseId'].isdigit():
                print('<paraphrase> attribute "wiktionarySenseId" has '
                      'non-integer value', paraphrase_dict['edited'])
            else:
                paraphrase_dict['wiktionarySenseId'] = \
                    int(paraphrase_dict['wiktionarySenseId'], 10)
            paraphrases.append(paraphrase_dict)
        else:
            print('unknown child of <wiktionaryParaphrases>', child)

    return paraphrases
1008710
def _maybe_download_corpus(tmp_dir, vocab_type):
  
  filename = os.path.basename(PTB_URL)
  compressed_filepath = generator_utils.maybe_download(
      tmp_dir, filename, PTB_URL)
  ptb_files = []
  ptb_char_files = []

  with tarfile.open(compressed_filepath, "r:gz") as tgz:
    files = []
    # Selecting only relevant files.
    for m in tgz.getmembers():
      if "ptb" in m.name and ".txt" in m.name:
        if "char" in m.name:
          ptb_char_files += [m.name]
        else:
          ptb_files += [m.name]
        files += [m]

    tgz.extractall(tmp_dir, members=files)

  if vocab_type == text_problems.VocabType.CHARACTER:
    return ptb_char_files
  else:
    return ptb_files
1079132
def get_preinit_encoders(encoders: List[encoder.Encoder]) -> List[Tuple[str, mx.init.Initializer]]:
    
    init = []  # type: List[Tuple[str, mx.init.Initializer]]
    for enc in encoders:
        if hasattr(enc, "get_initializers"):
            enc = cast(encoder_image.ImageLoadedCnnEncoder, enc)
            init.extend(enc.get_initializers())
    return init
1356764
def iter_tokens(cls, blob):
    
    readline_func = io.StringIO(blob.decode('utf-8')).readline
    return tokenize.generate_tokens(readline_func)
392639
def _ReadParserPresetsFromFile(self):
    
    self._presets_file = os.path.join(
        self._data_location, self._PRESETS_FILE_NAME)
    if not os.path.isfile(self._presets_file):
      raise errors.BadConfigOption(
          'No such parser presets file: {0:s}.'.format(self._presets_file))

    try:
      parsers_manager.ParsersManager.ReadPresetsFromFile(self._presets_file)
    except errors.MalformedPresetError as exception:
      raise errors.BadConfigOption(
          'Unable to read presets from file with error: {0!s}'.format(
              exception))
1265860
def _generate_relative_positions_embeddings(length_q, length_k, depth,
                                            max_relative_position, name,
                                            cache=False):
  """"""
  with tf.variable_scope(name):
    relative_positions_matrix = _generate_relative_positions_matrix(
        length_q, length_k, max_relative_position, cache=cache)
    vocab_size = max_relative_position * 2 + 1
    # Generates embedding for each relative position of dimension depth.
    embeddings_table = tf.get_variable("embeddings", [vocab_size, depth])
    embeddings = tf.gather(embeddings_table, relative_positions_matrix)
    return embeddings
983105
async def tuple(self, elem=None, elem_type=None, params=None):
        
        if hasattr(elem_type, 'blob_serialize'):
            container = elem_type() if elem is None else elem
            return await container.blob_serialize(self, elem=elem, elem_type=elem_type, params=params)

        if self.writing:
            return await self.dump_tuple(elem, elem_type, params)
        else:
            return await self.load_tuple(elem_type, params=params, elem=elem)