Generate the Skip-gram's pair.

        Args:
            token_list:     The list of tokens.

        Returns:
            zip of Tuple(Training N-gram data, Target N-gram data)
505623
def generate_skip_gram_data_set(self, token_list):
        '''
        
        '''
        n_gram_tuple_zip = self.generate_tuple_zip(token_list, 3)
        skip_gram_list = []
        for pre, point, post in n_gram_tuple_zip:
            skip_gram_list.append((point, pre))
            skip_gram_list.append((point, post))
        return zip(skip_gram_list)
840200
def _generate_tokens(self, text):
    
    # This is technically an undocumented API for Python3, but allows us to use the same API as for
    # Python2. See http://stackoverflow.com/a/4952291/328565.
    for index, tok in enumerate(tokenize.generate_tokens(io.StringIO(text).readline)):
      tok_type, tok_str, start, end, line = tok
      yield Token(tok_type, tok_str, start, end, line, index,
                  self._line_numbers.line_to_offset(start[0], start[1]),
                  self._line_numbers.line_to_offset(end[0], end[1]))
257611
def similar_filter_r(self, sentence_list):
        '''
        
        '''
        result_list = []
        recursive_list = []

        try:
            self.nlp_base.tokenize(sentence_list[0])
            subject_token = self.nlp_base.token
            result_list.append(sentence_list[0])
            if len(sentence_list) > 1:
                for i in range(len(sentence_list)):
                    if i > 0:
                        self.nlp_base.tokenize(sentence_list[i])
                        object_token = self.nlp_base.token
                        similarity = self.calculate(subject_token, object_token)
                        if similarity <= self.similarity_limit:
                            recursive_list.append(sentence_list[i])

            if len(recursive_list) > 0:
                result_list.extend(self.similar_filter_r(recursive_list))
        except IndexError:
            result_list = sentence_list

        return result_list
319600
def generate_clinamen(self, input_word, list_of_dict_words, swerve):
        
 
        results = []
        selected_list = []
        for i in list_of_dict_words: #produce a subset for efficency
            if len(i) < len(input_word)+1 and len(i) > len(input_word)/2:
                if '_' not in i:
                    selected_list.append(i)
        for i in selected_list:
            match = self.damerau_levenshtein_distance(input_word,i)
            if match == swerve:
                results.append(i)
        results = {'input' : input_word, 'results' : results, 'category' : 'clinamen'} 
        return results
1320730
def make_tokens(parse_buffer):
    
    tokens = []
    for token_type, result in parse_buffer:
        token = token_type(result)
        if token is not None:
            tokens.append(token)
    return tokens
295754
def chunks(self, chunk_size: int) -> Generator['TranslatorInput', None, None]:
        

        if len(self.tokens) > chunk_size and self.constraints is not None:
            logger.warning(
                'Input %s has length (%d) that exceeds max input length (%d), '
                'triggering internal splitting. Placing all target-side constraints '
                'with the first chunk, which is probably wrong.',
                self.sentence_id, len(self.tokens), chunk_size)

        for chunk_id, i in enumerate(range(0, len(self), chunk_size)):
            factors = [factor[i:i + chunk_size] for factor in self.factors] if self.factors is not None else None
            # Constrained decoding is not supported for chunked TranslatorInputs. As a fall-back, constraints are
            # assigned to the first chunk
            constraints = self.constraints if chunk_id == 0 else None
            pass_through_dict = self.pass_through_dict if chunk_id == 0 else None
            yield TranslatorInput(sentence_id=self.sentence_id,
                                  tokens=self.tokens[i:i + chunk_size],
                                  factors=factors,
                                  restrict_lexicon=self.restrict_lexicon,
                                  constraints=constraints,
                                  avoid_list=self.avoid_list,
                                  pass_through_dict=pass_through_dict)
741807
def tokens(self):
    
    seq = self.word_tokenizer.transform(Sequence(self.raw))
    return WordList(seq.tokens(), parent=self, language=self.language.code)
880173
def top_n_list(lang, n, wordlist='best', ascii_only=False):
    
    results = []
    for word in iter_wordlist(lang, wordlist):
        if (not ascii_only) or max(word) <= '~':
            results.append(word)
            if len(results) >= n:
                break
    return results
956916
def tabbed_parsing_token_generator(data_dir, tmp_dir, train, prefix,
                                   source_vocab_size, target_vocab_size):
  """"""
  filename = "parsing_{0}.pairs".format("train" if train else "dev")
  source_vocab = generator_utils.get_or_generate_tabbed_vocab(
      data_dir, tmp_dir, filename, 0,
      prefix + "_source.tokens.vocab.%d" % source_vocab_size, source_vocab_size)
  target_vocab = generator_utils.get_or_generate_tabbed_vocab(
      data_dir, tmp_dir, filename, 1,
      prefix + "_target.tokens.vocab.%d" % target_vocab_size, target_vocab_size)
  pair_filepath = os.path.join(tmp_dir, filename)
  return text_problems.text2text_generate_encoded(
      text_problems.text2text_txt_tab_iterator(pair_filepath), source_vocab,
      target_vocab)
1329160
def tokenlist(item, sep, suppress=True):
    """"""
    if suppress:
        sep = sep.suppress()
    return item + ZeroOrMore(sep + item) + Optional(sep)
987135
def token_list_to_text(tokenlist):
    
    ZeroWidthEscape = Token.ZeroWidthEscape
    return ''.join(item[1] for item in tokenlist if item[0] != ZeroWidthEscape)
876465
def completer_tokenize(cls, value, min_length=3):
        ''''''
        tokens = list(itertools.chain(*[
            [m for m in n.split("'") if len(m) > min_length]
            for n in value.split(' ')
        ]))
        return list(set([value] + tokens + [' '.join(tokens)]))
1045264
def get_tokens(max_value):
  
  vocab = [str(i) for i in range(max_value)]
  vocab = set(vocab)
  vocab.update(CodeOp.LITERALS)
  vocab.update(CodeOp.KEYWORDS)
  vocab |= set("".join(vocab))
  return sorted(vocab)
666058
def token_list_width(tokenlist):
    
    ZeroWidthEscape = Token.ZeroWidthEscape
    return sum(get_cwidth(c) for item in tokenlist for c in item[1] if item[0] != ZeroWidthEscape)
621265
def chunked_list(_list, _chunk_size=50):
    
    for i in range(0, len(_list), _chunk_size):
        yield _list[i:i + _chunk_size]
1340724
def tokens(self):
        
        for content in self.contents:
            if isinstance(content, TokenWithPosition):
                for word in content.split():
                    yield word
            else:
                yield content
436293
def collect_tokens(cls, parseresult, mode):
        
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                token = cls.recurse_token(token, inner)
                tokens[-1] = tokens[-1] + token
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens