Syllabify the given word.
761018
def _syllabify_simplex(word):
    ''''''
    word, rules = apply_T1(word)

    if re.search(r'[^ieAyOauo]*([ieAyOauo]{2})[^ieAyOauo]*', word):
        word, T2 = apply_T2(word)
        word, T8 = apply_T8(word)
        word, T9 = apply_T9(word)
        rules += T2 + T8 + T9

        # T4 produces variation
        syllabifications = list(apply_T4(word))

    else:
        syllabifications = [(word, ''), ]

    for word, rule in syllabifications:
        RULES = rules + rule

        if re.search(r'[ieAyOauo]{3}', word):
            word, T6 = apply_T6(word)
            word, T5 = apply_T5(word)
            word, T10 = apply_T10(word)
            word, T7 = apply_T7(word)
            word, T2 = apply_T2(word)
            RULES += T5 + T6 + T10 + T7 + T2

        RULES = RULES or ' T0'  # T0 means no rules have applied

        yield word, RULES
81829
def splitword(self, originalword, *newwords, **kwargs):
        """"""
        if isstring(originalword):
            originalword = self.doc[u(originalword)]
        return self.correctwords([originalword], newwords, **kwargs)
6140
def wrap(self, word, width, hyphen='-'):
        
        width -= len(hyphen)
        for w1, w2 in self.iterate(word):
            if len(w1) <= width:
                return w1 + hyphen, w2
1281503
def set_word_at_rva(self, rva, word):
        """"""
        return self.set_bytes_at_rva(rva, self.get_data_from_word(word))
466562
def spell(word):
    """"""
    w = Word(word)
    candidates = (common([word]) or exact([word]) or known([word]) or
                  known(w.typos()) or common(w.double_typos()) or
                  [word])
    correction = max(candidates, key=NLP_COUNTS.get)
    return get_case(word, correction)
328634
def apply_T4(word):  # OPTIMIZE
    ''''''
    WORD = _split_consonants_and_vowels(word)

    for k, v in WORD.iteritems():

        if len(v) == 2 and v.endswith(('u', 'y')):

            if WORD.get(k + 2, 0):

                if not WORD.get(k + 3, 0):
                    if len(WORD[k + 2]) == 1 and is_consonant(WORD[k + 2]):
                        WORD[k] = v[0] + '.' + v[1]

                elif len(WORD[k + 1]) == 1 and WORD.get(k + 3, 0):
                    if is_consonant(WORD[k + 3][0]):
                        WORD[k] = v[0] + '.' + v[1]

                elif len(WORD[k + 2]) == 2:
                    WORD[k] = v[0] + '.' + v[1]

    word = _compile_dict_into_word(WORD)

    return word
824428
def apply_T10(word):
    ''''''
    WORD = word
    offset = 0

    for iou in iou_sequences(WORD):
        i = iou.start(1) + 1 + offset
        WORD = WORD[:i] + '.' + WORD[i:]
        offset += 1

    RULE = ' T10' if word != WORD else ''

    return WORD, RULE
201659
def _load_words(self):
        """"""
        with open(self._words_file, 'r') as f:
            self._censor_list = [line.strip() for line in f.readlines()]
655822
def forward_word(self, e): # (M-f)
        """
        self.l_buffer.forward_word(self.argument_reset)
        self.finalize()
871555
def convert_vocab(vocab_file):
    """"""
    original_vocab = load_vocab(vocab_file)
    token_to_idx = dict(original_vocab)
    num_tokens = len(token_to_idx)
    idx_to_token = [None] * len(original_vocab)
    for word in original_vocab:
        idx = int(original_vocab[word])
        idx_to_token[idx] = word

    def swap(token, target_idx, token_to_idx, idx_to_token, swap_idx):
        original_idx = token_to_idx[token]
        original_token = idx_to_token[target_idx]
        token_to_idx[token] = target_idx
        token_to_idx[original_token] = original_idx
        idx_to_token[target_idx] = token
        idx_to_token[original_idx] = original_token
        swap_idx.append((original_idx, target_idx))

    reserved_tokens = [gluonnlp.vocab.BERTVocab.PADDING_TOKEN, gluonnlp.vocab.BERTVocab.CLS_TOKEN,
                       gluonnlp.vocab.BERTVocab.SEP_TOKEN, gluonnlp.vocab.BERTVocab.MASK_TOKEN]

    unknown_token = gluonnlp.vocab.BERTVocab.UNKNOWN_TOKEN
    padding_token = gluonnlp.vocab.BERTVocab.PADDING_TOKEN
    swap_idx = []
    assert unknown_token in token_to_idx
    assert padding_token in token_to_idx
    swap(unknown_token, 0, token_to_idx, idx_to_token, swap_idx)
    for i, token in enumerate(reserved_tokens):
        swap(token, i + 1, token_to_idx, idx_to_token, swap_idx)

    # sanity checks
    assert len(token_to_idx) == num_tokens
    assert len(idx_to_token) == num_tokens
    assert None not in idx_to_token
    assert len(set(idx_to_token)) == num_tokens

    bert_vocab_dict = {}
    bert_vocab_dict['idx_to_token'] = idx_to_token
    bert_vocab_dict['token_to_idx'] = token_to_idx
    bert_vocab_dict['reserved_tokens'] = reserved_tokens
    bert_vocab_dict['unknown_token'] = unknown_token
    bert_vocab_dict['padding_token'] = padding_token
    bert_vocab_dict['bos_token'] = None
    bert_vocab_dict['eos_token'] = None
    bert_vocab_dict['mask_token'] = gluonnlp.vocab.BERTVocab.MASK_TOKEN
    bert_vocab_dict['sep_token'] = gluonnlp.vocab.BERTVocab.SEP_TOKEN
    bert_vocab_dict['cls_token'] = gluonnlp.vocab.BERTVocab.CLS_TOKEN
    json_str = json.dumps(bert_vocab_dict)
    converted_vocab = gluonnlp.vocab.BERTVocab.from_json(json_str)
    return converted_vocab, swap_idx
108014
def transcribe(self, text, as_phonemes = False):
		'''
		
		'''
		phoneme_words = [self.transcribe_word(word) for word in self._tokenize(text)]
		if not as_phonemes:
			words = [''.join([phoneme.ipa for phoneme in word]) for word in phoneme_words]
			return ' '.join(words)
		else:
			return phoneme_words
1154859
def keyword(self, text):
        """"""
        cls = self.KEYWORDS[text]
        self.push_token(cls(text, self.lineno, self.offset))
1266667
def split_into_words(s):
  """"""
  s = re.sub(r"\W+", " ", s)
  s = re.sub(r"[_0-9]+", " ", s)
  return s.split()
412584
def transcribe_to_modern(self, text) :
		'''
		
		'''
		# first transcribe letter by letter
		phoneme_words = self.transcribe(text, as_phonemes = True)
		words = [''.join([self.to_modern[0][phoneme.ipa] for phoneme in word]) for word in phoneme_words]
		modern_text =  ' '.join(words)

		# then apply phonotactic fixes
		for regexp, replacement in self.to_modern[1]:
			modern_text = re.sub(regexp, replacement, modern_text)

		return modern_text
489590
def same_syllabic_feature(ch1, ch2):
    ''''''
    if ch1 == '.' or ch2 == '.':
        return False

    ch1 = 'V' if ch1 in VOWELS else 'C' if ch1 in CONSONANTS else None
    ch2 = 'V' if ch2 in VOWELS else 'C' if ch2 in CONSONANTS else None

    return ch1 == ch2
1151540
def syllable_tokenize(text: str) -> List[str]:
    

    if not text or not isinstance(text, str):
        return []

    tokens = []
    if text:
        words = word_tokenize(text)
        trie = dict_trie(dict_source=thai_syllables())
        for word in words:
            tokens.extend(word_tokenize(text=word, custom_dict=trie))

    return tokens
475498
def convert_consonantal_i(self, word) -> str:
        """"""
        match = list(self.consonantal_i_matcher.finditer(word))
        if match:
            if word[0].isupper():
                return "J" + word[1:]
            return "j" + word[1:]
        return word