builds a Tamil bigram letter model
172899
def language_model(self,verbose=True):
        
        # use a generator in corpus
        prev = None
        for next_letter in self.corpus.next_tamil_letter():
            # update frequency from corpus
            if prev:
                self.letter2[prev][next_letter] += 1
                if ( verbose ) :
                    print(prev)
                    print(next_letter)
                    print( self.letter2[prev][next_letter] )
            prev = next_letter #update always
        return
1250121
def build_char_states(self, char_embed, is_training, reuse, char_ids, char_lengths):
        """"""
        max_char_length = self.cfg.max_char_length

        inputs = dropout(tf.nn.embedding_lookup(char_embed, char_ids),
                         self.cfg.dropout, is_training)
        inputs = tf.reshape(
            inputs, shape=[max_char_length, -1, self.cfg.char_embed_dim])
        char_lengths = tf.reshape(char_lengths, shape=[-1])
        with tf.variable_scope('char_encoding', reuse=reuse):
            cell_fw = XGRUCell(hidden_dim=self.cfg.char_embed_dim)
            cell_bw = XGRUCell(hidden_dim=self.cfg.char_embed_dim)
            _, (left_right, right_left) = tf.nn.bidirectional_dynamic_rnn(
                cell_fw=cell_fw,
                cell_bw=cell_bw,
                sequence_length=char_lengths,
                inputs=inputs,
                time_major=True,
                dtype=tf.float32
            )

        left_right = tf.reshape(left_right, shape=[-1, self.cfg.char_embed_dim])

        right_left = tf.reshape(right_left, shape=[-1, self.cfg.char_embed_dim])

        states = tf.concat([left_right, right_left], axis=1)
        out_shape = tf.shape(char_ids)[1:3]
        out_shape = tf.concat([out_shape, tf.constant(
            value=[self.cfg.char_embed_dim * 2], dtype=tf.int32)], axis=0)
        return tf.reshape(states, shape=out_shape)
471004
def BiLSTM(nO, nI):
    """"""
    return Bidirectional(LSTM(nO // 2, nI), LSTM(nO // 2, nI))
1294855
def rnn(bptt, vocab_size, num_embed, nhid, num_layers, dropout, num_proj, batch_size):
    
    state_names = []
    data = S.var('data')
    weight = S.var("encoder_weight", stype='row_sparse')
    embed = S.sparse.Embedding(data=data, weight=weight, input_dim=vocab_size,
                               output_dim=num_embed, name='embed', sparse_grad=True)
    states = []
    outputs = S.Dropout(embed, p=dropout)
    for i in range(num_layers):
        prefix = 'lstmp%d_' % i
        init_h = S.var(prefix + 'init_h', shape=(batch_size, num_proj), init=mx.init.Zero())
        init_c = S.var(prefix + 'init_c', shape=(batch_size, nhid), init=mx.init.Zero())
        state_names += [prefix + 'init_h', prefix + 'init_c']
        lstmp = mx.gluon.contrib.rnn.LSTMPCell(nhid, num_proj, prefix=prefix)
        outputs, next_states = lstmp.unroll(bptt, outputs, begin_state=[init_h, init_c], \
                                            layout='NTC', merge_outputs=True)
        outputs = S.Dropout(outputs, p=dropout)
        states += [S.stop_gradient(s) for s in next_states]
    outputs = S.reshape(outputs, shape=(-1, num_proj))

    trainable_lstm_args = []
    for arg in outputs.list_arguments():
        if 'lstmp' in arg and 'init' not in arg:
            trainable_lstm_args.append(arg)
    return outputs, states, trainable_lstm_args, state_names
1243333
def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES):
    '''
    
    '''
    layers = [
        Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape),
        Conv2D(64, (3, 3), activation='relu'),
        MaxPooling2D(pool_size=(2, 2)),
        Flatten(),
        Dense(100, activation='relu'),
        Dense(num_classes, activation='softmax')
    ]

    model = Sequential(layers)

    if hyper_params['optimizer'] == 'Adam':
        optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate'])
    else:
        optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9)
    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])

    return model
1148745
def main(model=None, output_dir=None, n_iter=100):
    """"""
    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank("en")  # create blank Language class
        print("Created blank 'en' model")

    # create the built-in pipeline components and add them to the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    if "ner" not in nlp.pipe_names:
        ner = nlp.create_pipe("ner")
        nlp.add_pipe(ner, last=True)
    # otherwise, get it so we can add labels
    else:
        ner = nlp.get_pipe("ner")

    # add labels
    for _, annotations in TRAIN_DATA:
        for ent in annotations.get("entities"):
            ner.add_label(ent[2])

    # get names of other pipes to disable them during training
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "ner"]
    with nlp.disable_pipes(*other_pipes):  # only train NER
        # reset and initialize the weights randomly – but only if we're
        # training a new model
        if model is None:
            nlp.begin_training()
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            # batch up the examples using spaCy's minibatch
            batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
            for batch in batches:
                texts, annotations = zip(*batch)
                nlp.update(
                    texts,  # batch of texts
                    annotations,  # batch of annotations
                    drop=0.5,  # dropout - make it harder to memorise data
                    losses=losses,
                )
            print("Losses", losses)

    # test the trained model
    for text, _ in TRAIN_DATA:
        doc = nlp(text)
        print("Entities", [(ent.text, ent.label_) for ent in doc.ents])
        print("Tokens", [(t.text, t.ent_type_, t.ent_iob) for t in doc])

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the saved model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        for text, _ in TRAIN_DATA:
            doc = nlp2(text)
            print("Entities", [(ent.text, ent.label_) for ent in doc.ents])
            print("Tokens", [(t.text, t.ent_type_, t.ent_iob) for t in doc])
1334718
def _build_unicode_character_database(self):
        
        filename = "UnicodeData.txt"
        current_dir = os.path.abspath(os.path.dirname(__file__))
        tag = re.compile(r"<\w+?>")
        with codecs.open(os.path.join(current_dir, filename), mode="r", encoding="utf-8") as fp:
            for line in fp:
                if not line.strip():
                    continue
                data = line.strip().split(";")
                # Replace the start/end range markers with their proper derived names.
                if data[1].endswith((u"First>", u"Last>")) and _is_derived(int(data[0], 16)):
                    data[1] = _get_nr_prefix(int(data[0], 16))
                    if data[1].startswith("HANGUL SYLLABLE"):  # For Hangul syllables, use naming rule NR1
                        data[1] += _get_hangul_syllable_name(int(data[0], 16))
                    else:  # Others should use naming rule NR2
                        data[1] += data[0]
                data[3] = int(data[3])  # Convert the Canonical Combining Class value into an int.
                if data[5]:  # Convert the contents of the decomposition into characters, preserving tag info.
                    data[5] = u" ".join([_hexstr_to_unichr(s) if not tag.match(s) else s for s in data[5].split()])
                for i in [6, 7, 8]:  # Convert the decimal, digit and numeric fields to either ints or fractions.
                    if data[i]:
                        if "/" in data[i]:
                            data[i] = Fraction(data[i])
                        else:
                            data[i] = int(data[i])
                for i in [12, 13, 14]:  # Convert the uppercase, lowercase and titlecase fields to characters.
                    if data[i]:
                        data[i] = _hexstr_to_unichr(data[i])
                lookup_name = _uax44lm2transform(data[1])
                uc_data = UnicodeCharacter(u"U+" + data[0], *data[1:])
                self._unicode_character_database[int(data[0], 16)] = uc_data
                self._name_database[lookup_name] = uc_data
        # Fill out the "compressed" ranges of UnicodeData.txt i.e. fill out the remaining characters per the Name
        # Derivation Rules.  See the Unicode Standard, ch. 4, section 4.8, Unicode Name Property
        for lookup_range, prefix_string in _nr_prefix_strings.items():
            exemplar = self._unicode_character_database.__getitem__(lookup_range[0])
            for item in lookup_range:
                hex_code = _padded_hex(item)
                new_name = prefix_string
                if prefix_string.startswith("HANGUL SYLLABLE"):  # For Hangul, use naming rule NR1
                    new_name += _get_hangul_syllable_name(item)
                else:  # Everything else uses naming rule NR2
                    new_name += hex_code
                uc_data = exemplar._replace(code=u"U+" + hex_code, name=new_name)
                self._unicode_character_database[item] = uc_data
                self._name_database[_uax44lm2transform(new_name)] = uc_data
1195604
def make_pattern( patt, flags=0 ):
    
    # print('input',len(patt))
    patt_letters = utf8.get_letters( patt )
    patt_out = list()
    idx = 0
    # print('output',len(patt_letters))
    patt = [None,None]
    prev = None
    LEN_PATT = len(patt_letters)
    while( idx < LEN_PATT ):
        if utf8.istamil( patt_letters[idx] ) and ( prev == '-' or ((idx+1) < LEN_PATT and patt_letters[idx+1] == u'-') ):
            if (idx+1) < LEN_PATT and patt_letters[idx+1] == u'-':
                patt[0] = patt_letters[idx]
                idx = idx + 2
                prev = "-"
            elif prev == '-':
                patt[1] = patt_letters[idx]
                patt_out.extend( expand_tamil( patt[0], patt[1]) )
                idx = idx + 1
                prev = patt_letters[idx]
            continue
        patt_out.extend( patt_letters[idx] )
        prev = patt_letters[idx]
        idx = idx + 1
    opattern = u"".join( patt_out )
    compile_regexp = re.compile( opattern, flags )        
    return (compile_regexp,opattern)
340981
def num2tamilstr( *args ):
    
    number = args[0]
    if len(args) < 2:
        filenames = []
    else:
        filenames = args[1]
    if len(args) ==3:
        tensSpecial = args[2]
    else:
        tensSpecial='BASIC'
    if not any( filter( lambda T: isinstance( number, T), [str,unicode,int, long, float]) ) or isinstance(number,complex):
        raise Exception('num2tamilstr input has to be a long or integer or float')
    if float(number) > long(1e12):
        raise Exception('num2tamilstr input is too large')
    if float(number) < 0:
        return u"- "+num2tamilstr( -number )

    units = (u'பூஜ்ஜியம்', u'ஒன்று', u'இரண்டு', u'மூன்று', u'நான்கு', u'ஐந்து', u'ஆறு', u'ஏழு', u'எட்டு', u'ஒன்பது', u'பத்து') # 0-10
    units_suffix = (u'பூஜ்ஜியம்', u'தொன்று', u'திரண்டு', u'மூன்று', u'நான்கு', u'தைந்து', u'தாறு', u'தேழு', u'தெட்டு', u'தொன்பது', u'பத்து') # 0-10
    units_suffix_nine = (u'பூஜ்ஜியம்', u'றொன்று', u'றிரண்டு', u'மூன்று', u'நான்கு', u'றைந்து', u'றாறு', u'றேழு', u'றெட்டு', u'றொன்பது', u'பத்து') # 0-10
    tween = [1.0,2.0,5.0,6.0,7.0,8.0,9.0]
    teens = (u'பதினொன்று', u'பனிரண்டு', u'பதிமூன்று', u'பதினான்கு', u'பதினைந்து',u'பதினாறு', u'பதினேழு', u'பதினெட்டு', u'பத்தொன்பது') # 11-19
    tens = (u'பத்து', u'இருபது', u'முப்பது', u'நாற்பது', u'ஐம்பது',u'அறுபது', u'எழுபது', u'எண்பது', u'தொன்னூறு') # 10-90
    tens_full_prefix = (u'இருபத்து', u'முப்பத்து', u'நாற்பத்து', u'ஐம்பத்து', u'அறுபத்து', u'எழுபத்து', u'எண்பத்து', u'தொன்னூற்று') # 10+-90+
    tens_prefix = (u'இருபத்', u'முப்பத்', u'நாற்பத்', u'ஐம்பத்', u'அறுபத்', u'எழுபத்', u'எண்பத்', u'தொன்னூற்') # 10+-90+
    hundreds = ( u'நூறு', u'இருநூறு', u'முன்னூறு', u'நாநூறு',u'ஐநூறு', u'அறுநூறு', u'எழுநூறு', u'எண்ணூறு', u'தொள்ளாயிரம்') #100 - 900
    hundreds_suffix = (u'நூற்றி', u'இருநூற்று', u'முன்னூற்று', u'நாநூற்று', u'ஐநூற்று', u'அறுநூற்று', u'எழுநூற்று', u'எண்ணூற்று',u'தொள்ளாயிரத்து') #100+ - 900+

    one_thousand_prefix = u'ஓர்'
    thousands = (u'ஆயிரம்',u'ஆயிரத்து')

    one_prefix = u'ஒரு'
    lakh = (u'இலட்சம்',u'இலட்சத்து')
    crore = (u'கோடி',u'கோடியே')

    pulli = u'புள்ளி'
    n_one = 1.0
    n_ten = 10.0
    n_hundred = 100.0
    n_thousand = 1000.0
    n_lakh = 100.0*n_thousand
    n_crore = (100.0*n_lakh)
    # handle fractional parts
    if float(number) > 0.0 and float(number) < 1.0:
        rval = []
        rval.append(pulli)
        filenames.append( 'pulli' )
        number_str = str(number).replace('0.','')
        for digit in number_str:
            filenames.append( "units_%d"%int(digit))
            rval.append( units[int(digit)] )
        return u' '.join(rval)

    if isinstance(number,str) or isinstance(number,unicode):
        result = u""
        number = number.strip()
        assert(len(args) == 1)
        assert(len(number) > 0)
        is_negative = number[0] == "-"
        if  is_negative:
            number = number[1:]
        frac_part = u""
        if number.find(".") >= 0:
            rat_part,frac_part = number.split(".")
            frac_part = num2tamilstr(u"0."+frac_part)
        else:
            rat_part = number
        if len(rat_part) > 0:
            result = num2tamilstr(float(rat_part))
        result = result +u" "+ frac_part
        return is_negative and "-" + result.strip() or result.strip()

    suffix_base = { n_crore: crore, n_lakh : lakh, n_thousand : thousands}
    suffix_file_map = {n_crore: "crore", n_lakh : "lakh", n_thousand : "thousands"}

    file_map = {n_crore :["one_prefix","crore_0"],
                n_lakh : ["one_prefix","lakh_0"],
                n_thousand :  ["one_thousand_prefix", "thousands_0"],
               n_hundred : ["hundreds_0"], #special
               n_ten : ["units_10"],
               n_one : ["units_1"]}

    num_map = {n_crore : [one_prefix,crore[0]],
               n_lakh  : [one_prefix,lakh[0]],
               n_thousand : [one_thousand_prefix, thousands[0]],
               n_hundred : [hundreds[0]], #special
               n_ten : [units[10]],
               n_one : [units[1]]}

    all_bases = [n_crore, n_lakh, n_thousand, n_hundred, n_ten,n_one]
    allowed_bases = list(filter( lambda base: number >= base, all_bases ))
    if len(allowed_bases) >= 1:
        n_base = allowed_bases[0]
        if number == n_base:
            if tensSpecial=='BASIC':
                filenames.extend(file_map[n_base])
                return u" ".join(num_map[n_base])
            elif tensSpecial=='NINES':
                filenames.extend(file_map[n_base])
                return units_suffix_nine[long(number%10)]
            else:
                filenames.extend(file_map[n_base])
                return units_suffix[long(number%10)]
        quotient_number = long( number/n_base )
        residue_number = number - n_base*quotient_number
        #print number, n_base, quotient_number, residue_number, tensSpecial
        if n_base == n_one:
            if isinstance(number,float):
                int_part = long(number%10)
                frac = number - float(int_part)
                filenames.append("units_%d"%int_part)
                if abs(frac) > 1e-30:
                    if tensSpecial=='BASIC':
                        return units[int_part]+u' ' + num2tamilstr(frac,filenames)
                    elif tensSpecial=='NINES':
                        return units_suffix_nine[int_part]+u' ' + num2tamilstr(frac,filenames)
                    else:
                        return units_suffix[int_part]+u' ' + num2tamilstr(frac,filenames)

                else:
                    if tensSpecial=='BASIC':
                        return units[int_part]
                    elif tensSpecial=='NINES':
                        return units_suffix_nine[int_part]
                    else:
                        return units_suffix[int_part]
            else:
                if tensSpecial=='BASIC':
                    filenames.append("units_%d"%number)
                    return units[number]
                elif tensSpecial=='NINES':
                    filenames.append("units_%d"%number)
                    return units_suffix_nine[number]
                else:
                    filenames.append("units_%d"%number)
                    return units_suffix[number]

        elif n_base == n_ten:
            if residue_number < 1.0:
                filenames.append("tens_%d"%(quotient_number-1))
                if residue_number == 0.0:
                    return tens[quotient_number-1]
                #else: //seems not reachable.
                #    numeral = tens[quotient_number-1]
            elif number < 20:
                filenames.append("teens_%d"%(number-10))
                residue_number = math.fmod(number,1)
                teen_number = int(math.floor(number - 10))
                if residue_number > 1e-30:
                    return teens[teen_number-1] +u' ' + num2tamilstr(residue_number,filenames)
                else:
                    return teens[teen_number-1]+u' '
            if residue_number < 1.0:
                filenames.append( "tens_%d"%(quotient_number-1) )
                numeral = tens[quotient_number-1]+u' '
            else:
                if residue_number in tween:
                    filenames.append( "tens_prefix_%d"%(quotient_number-2) )
                    numeral = tens_prefix[quotient_number-2]
                    tensSpecial='SPECIAL'
                    if (quotient_number==9):
                        tensSpecial = 'NINES'
                else:
                    filenames.append( "tens_prefix_%d"%(quotient_number-2) )
                    numeral = tens_full_prefix[quotient_number-2]+u' '
        elif n_base == n_hundred:
            if residue_number == 0:
                filenames.append("hundreds_%d"%(quotient_number-1))
                return hundreds[quotient_number-1]+u' '
            if residue_number < 1.0:
                filenames.append( "hundreds_%d"%(quotient_number-1) )
                numeral = hundreds[quotient_number-1]+u' '
            else:
                filenames.append("hundreds_suffix_%d"%(quotient_number-1))
                numeral = hundreds_suffix[quotient_number-1]+u' '
        else:
            if ( quotient_number == 1 ):
                if n_base == n_thousand:
                    filenames.append("one_thousand_prefix")
                    numeral = one_thousand_prefix
                else:
                    filenames.append("one_prefix")
                    numeral = one_prefix
            else:
                numeral = num2tamilstr( quotient_number, filenames )
        if n_base >= n_thousand:
            suffix = suffix_base[n_base][long(residue_number >= 1)]
            suffix_filename = "%s_%d"%(suffix_file_map[n_base],long(residue_number >= 1))
            if residue_number == 0:
                filenames.append(suffix_filename)
                return numeral + u' ' + suffix+u' '
            filenames.append(suffix_filename)
            numeral = numeral + u' ' + suffix+u' '
        residue_numeral = num2tamilstr( residue_number, filenames, tensSpecial)
        #return numeral+u' '+residue_numeral
        return numeral+residue_numeral
    # number has to be zero
    filenames.append("units_0")
    return units[0]
595295
def morphemes(args):
  """"""
  morfessor = load_morfessor_model(lang=args.lang)
  for l in args.input:
    words = l.strip().split()
    morphemes = [(w, u"_".join(morfessor.viterbi_segment(w)[0])) for w in words]
    line_annotations = [u"{:<16}{:<5}".format(w,p) for w, p in morphemes]
    _print(u"\n".join(line_annotations))
    _print(u"")
31790
def _JMS_to_Bern_I(C, qq):
    """'"""
    if qq in ['sb', 'db', 'ds']:
        dd = 'dd'
        ij = tuple(dflav[q] for q in qq)
    elif qq == 'cu':
        dd = 'uu'
        ij = tuple(uflav[q] for q in qq)
    else:
        raise ValueError("not in Bern_I: ".format(qq))
    ji = (ij[1], ij[0])
    d = {
        '1' + 2 * qq : C["V{}LL".format(dd)][ij + ij],
        '2' + 2 * qq : C["S1{}RR".format(dd)][ji + ji].conj()
                       - C["S8{}RR".format(dd)][ji + ji].conj() / (2 * Nc),
        '3' + 2 * qq : C["S8{}RR".format(dd)][ji + ji].conj() / 2,
        '4' + 2 * qq : -C["V8{}LR".format(dd)][ij + ij],
        '5' + 2 * qq : -2 * C["V1{}LR".format(dd)][ij + ij]
                       + C["V8{}LR".format(dd)][ij + ij] / Nc,
        '1p' + 2 * qq : C["V{}RR".format(dd)][ij + ij],
        '2p' + 2 * qq : C["S1{}RR".format(dd)][ij + ij]
                       - C["S8{}RR".format(dd)][ij + ij] / (2 * Nc),
        '3p' + 2 * qq : C["S8{}RR".format(dd)][ij + ij] / 2
            }
    return d
884530
def ngram2mgram(input_file, output_file, n, m, words=False, ascii_idngram=False):
    
    cmd = ['ngram2mgram', '-n', n,
                          '-m', m]

    if words and ascii_idngram:
        raise ConversionError("Parameters 'words' and 'ascii_idngram' cannot both be True")

    if words:
        cmd.append('-words')
    elif ascii_idngram:
        cmd.append('-ascii')
    else:
        cmd.append('-binary')

    # Ensure that every parameter is of type 'str'
    cmd = [str(x) for x in cmd]

    with open(input_file,'r') as input_f:
        with open(output_file,'w+') as output_f:
            with  output_to_debuglogger() as err_f:
                exitcode = subprocess.call(cmd, stdin=input_f, stdout=output_f, stderr=err_f)
    
    logger = logging.getLogger(__name__)
    logger.debug("Command '%s' returned with exit code '%d'." % (' '.join(cmd), exitcode))

    if exitcode != 0:
        raise ConversionError("'%s' returned with non-zero exit status '%s'" % (cmd[0], exitcode))
98683
def cbow_fasttext_batch(centers, contexts, num_tokens, subword_lookup, dtype,
                        index_dtype):
    """"""
    _, contexts_row, contexts_col = contexts
    data, row, col = subword_lookup(contexts_row, contexts_col)
    centers = mx.nd.array(centers, dtype=index_dtype)
    contexts = mx.nd.sparse.csr_matrix(
        (data, (row, col)), dtype=dtype,
        shape=(len(centers), num_tokens))  # yapf: disable
    return centers, contexts
1006716
def generate_ngrams(args, parser):
    """"""
    store = utils.get_data_store(args)
    corpus = utils.get_corpus(args)
    if args.catalogue:
        catalogue = utils.get_catalogue(args)
    else:
        catalogue = None
    store.add_ngrams(corpus, args.min_size, args.max_size, catalogue)
258540
def create_ngram_set(input_list, ngram_value=2):
    
    return set(zip(*[input_list[i:] for i in range(ngram_value)]))
1326335
def orthographic_syllabify(self, word):
        """"""
        p_vectors = [self.get_phonetic_feature_vector(c, self.lang) for c in word]

        syllables = []

        for i in range(len(word)):
            v = p_vectors[i]

            syllables.append(word[i])

            if i + 1 < len(word) and (not self.is_valid(p_vectors[i + 1]) or self.is_misc(p_vectors[i + 1])):
                syllables.append(u' ')

            elif not self.is_valid(v) or self.is_misc(v):
                syllables.append(u' ')

            elif self.is_vowel(v):

                anu_nonplos = (i + 2 < len(word) and
                               self.is_anusvaar(p_vectors[i + 1]) and
                               not self.is_plosive(p_vectors[i + 2])
                               )

                anu_eow = (i + 2 == len(word) and
                           self.is_anusvaar(p_vectors[i + 1]))

                if not (anu_nonplos or anu_eow):
                    syllables.append(u' ')

            elif i + 1 < len(word) and (self.is_consonant(v) or self.is_nukta(v)):
                if self.is_consonant(p_vectors[i + 1]):
                    syllables.append(u' ')
                elif self.is_vowel(p_vectors[i + 1]) and not self.is_dependent_vowel(p_vectors[i + 1]):
                    syllables.append(u' ')
                elif self.is_anusvaar(p_vectors[i + 1]):
                    anu_nonplos = (i + 2 < len(word) and not self.is_plosive(p_vectors[i + 2]))

                    anu_eow = i + 2 == len(word)

                    if not (anu_nonplos or anu_eow):
                        syllables.append(u' ')

        return u''.join(syllables).strip().split(u' ')
49927
private double[] detectBlockLongText(List<String> ngrams) {
        assert !ngrams.isEmpty();
        double[] langprob = new double[ngramFrequencyData.getLanguageList().size()];
        Random rand = new Random(seed.or(DEFAULT_SEED));
        for (int t = 0; t < N_TRIAL; ++t) {
            double[] prob = initProbability();
            double alpha = this.alpha + (rand.nextGaussian() * ALPHA_WIDTH);

            for (int i=0; i<ITERATION_LIMIT; i++) {
                int r = rand.nextInt(ngrams.size());
                updateLangProb(prob, ngrams.get(r), 1, alpha);
                if (i % 5 == 0) {
                    if (Util.normalizeProb(prob) > CONV_THRESHOLD) break; //this break ensures that we quit the loop before all probabilities reach 0
                    if (logger.isTraceEnabled()) logger.trace("> " + sortProbability(prob));
                }
            }
            for(int j=0;j<langprob.length;++j) langprob[j] += prob[j] / N_TRIAL;
            if (logger.isDebugEnabled()) logger.debug("==> " + sortProbability(prob));
        }
        return langprob;
    }