Phontic transcription of the ShortLine instances.
        :param transcriber:
        :return:
348292
def to_phonetics(self, transcriber):
        
        for viisuordh in tokenize_old_norse_words(self.text):
            word = normalize(viisuordh)
            if word:
                transcribed_word = transcriber.text_to_phonetic_representation(word)
                pfl = transcriber.text_to_phonemes(word)

                self.transcribed.append(transcribed_word)
                self.phonological_features_text.append(pfl)
        self.get_first_sounds()
238236
def splitter_support(py2enc):
    '''
    '''
    if sys.version < '3':
        def _fn_sentence(pattern, sentence):
            if REGEXTYPE == type(pattern):
                if pattern.flags & re.UNICODE:
                    return sentence.decode(py2enc)
                else:
                    return sentence
            else:
                return sentence
        def _fn_token2str(pattern):
            if REGEXTYPE == type(pattern):
                if pattern.flags & re.UNICODE:
                    def _fn(token):
                        return token.encode(py2enc)
                else:
                    def _fn(token):
                        return token
            else:
                def _fn(token):
                    return token
            return _fn
    else:
        def _fn_sentence(pattern, sentence):
            return sentence
        def _fn_token2str(pattern):
            def _fn(token):
                return token
            return _fn

    def _fn_tokenize_pattern(text, pattern):
        pos = 0
        sentence = _fn_sentence(pattern, text)
        postprocess = _fn_token2str(pattern)
        for m in re.finditer(pattern, sentence):
            if pos < m.start():
                token = postprocess(sentence[pos:m.start()])
                yield (token.strip(), False)
                pos = m.start()
            token = postprocess(sentence[pos:m.end()])
            yield (token.strip(), True)
            pos = m.end()
        if pos < len(sentence):
            token = postprocess(sentence[pos:])
            yield (token.strip(), False)

    def _fn_tokenize_features(text, features):
        acc = []
        acc.append((text.strip(), False))

        for feat in features:
            for i,e in enumerate(acc):
                if e[1]==False:
                    tmp = list(_fn_tokenize_pattern(e[0], feat))
                    if len(tmp) > 0:
                        acc.pop(i)
                        acc[i:i] = tmp
        return acc
                        
    return _fn_tokenize_pattern, _fn_tokenize_features
795392
def convert_examples_to_features(examples, label_list, max_seq_length,
                                 tokenizer, output_mode):
    """"""

    label_map = {label : i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d of %d" % (ex_index, len(examples)))

        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambiguously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        segment_ids = [0] * len(tokens)

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            segment_ids += [1] * (len(tokens_b) + 1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        if output_mode == "classification":
            label_id = label_map[example.label]
        elif output_mode == "regression":
            label_id = float(example.label)
        else:
            raise KeyError(output_mode)

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % (example.guid))
            logger.info("tokens: %s" % " ".join(
                    [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
            logger.info("label: %s (id = %d)" % (example.label, label_id))

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_mask=input_mask,
                              segment_ids=segment_ids,
                              label_id=label_id))
    return features
755354
def tokenize(qp_pair, tokenizer=None, is_training=False):
    '''
    
    '''
    question_tokens = tokenizer.tokenize(qp_pair['question'])
    passage_tokens = tokenizer.tokenize(qp_pair['passage'])
    if is_training:
        question_tokens = question_tokens[:300]
        passage_tokens = passage_tokens[:300]
    passage_tokens.insert(
        0, {'word': '<BOS>', 'original_text': '<BOS>', 'char_begin': 0, 'char_end': 0})
    passage_tokens.append(
        {'word': '<EOS>', 'original_text': '<EOS>', 'char_begin': 0, 'char_end': 0})
    qp_pair['question_tokens'] = question_tokens
    qp_pair['passage_tokens'] = passage_tokens
408774
def read_until(self, terminator):
        

        lines = []
        term = terminator + b'\n'
        while True:
            line = self.input.readline()
            if line == term:
                break
            else:
                lines.append(line)
        return b''.join(lines)
1363502
def nltk_tokenzier_factory(nltk_tokenizer):
	'''
	
	'''

	def tokenize(text):
		toks = []
		for tok in nltk_tokenizer.tokenize(text):
			if len(tok) > 0:
				toks.append(Tok(_get_pos_tag(tok),
				                tok.lower(),
				                tok.lower(),
				                ent_type='',
				                tag=''))
		return Doc([toks], text)

	return tokenize
959811
def _create_sentence_objects(self):
    '''
    '''
    sentence_objects = []
    sent_tokenizer = SentenceTokenizer(locale=self.language.code)
    seq = Sequence(self.raw)
    seq = sent_tokenizer.transform(seq)
    for start_index, end_index in zip(seq.idx[:-1], seq.idx[1:]):
      # Sentences share the same models as their parent blob
      sent = seq.text[start_index: end_index].strip()
      if not sent: continue
      s = Sentence(sent, start_index=start_index, end_index=end_index)
      s.detected_languages = self.detected_languages
      sentence_objects.append(s)
    return sentence_objects
1091562
def gen_docs(corpus, lemmatize, rm_stops):
    

    assert corpus in ['phi5', 'tlg']

    if corpus == 'phi5':
        language = 'latin'
        filepaths = assemble_phi5_author_filepaths()
        jv_replacer = JVReplacer()
        text_cleaner = phi5_plaintext_cleanup
        word_tokenizer = nltk_tokenize_words
        if rm_stops:
            stops = latin_stops
        else:
            stops = None
    elif corpus == 'tlg':
        language = 'greek'
        filepaths = assemble_tlg_author_filepaths()
        text_cleaner = tlg_plaintext_cleanup
        word_tokenizer = nltk_tokenize_words

        if rm_stops:
            stops = latin_stops
        else:
            stops = None

    if lemmatize:
        lemmatizer = LemmaReplacer(language)

    sent_tokenizer = TokenizeSentence(language)

    for filepath in filepaths:
        with open(filepath) as f:
            text = f.read()
        # light first-pass cleanup, before sentence tokenization (which relies on punctuation)
        text = text_cleaner(text, rm_punctuation=False, rm_periods=False)
        sent_tokens = sent_tokenizer.tokenize_sentences(text)
        # doc_sentences = []
        for sentence in sent_tokens:
            # a second cleanup at sentence-level, to rm all punctuation
            sentence = text_cleaner(sentence, rm_punctuation=True, rm_periods=True)
            sentence = word_tokenizer(sentence)
            sentence = [s.lower() for s in sentence]
            sentence = [w for w in sentence if w]
            if language == 'latin':
                sentence = [w[1:] if w.startswith('-') else w for w in sentence]

            if stops:
                sentence = [w for w in sentence if w not in stops]

            sentence = [w for w in sentence if len(w) > 1]  # rm short words

            if sentence:
                sentence = sentence

            if lemmatize:
                sentence = lemmatizer.lemmatize(sentence)
            if sentence and language == 'latin':
                sentence = [jv_replacer.replace(word) for word in sentence]
            if sentence:
                yield sentence
247026
def _codon_positions(self, feature):
        
        results = self._transcript_feature_positions(feature)
        if len(results) != 3:
            raise ValueError(
                "Expected 3 positions for %s of %s but got %d" % (
                    feature,
                    self.id,
                    len(results)))
        return results
1095798
def _strfactory(cls, line):
        """"""

        cmp = line.rstrip().split()
        chrom = cmp[2]
        if not chrom.startswith("chr"):
            chrom = "chr%s" % chrom
        instance =  tuple.__new__(cls,
                (cmp[0], cmp[1],
                    chrom, int(cmp[3]), int(cmp[4]),
                    {'1' : '+', '-1' : '-'}[cmp[5]], cmp[6]))
        span = instance.end - instance.start + 1
        m_num = sum( (t[1] == "M" and [t[0]] or [0])[0] for t in instance.cigar_iter(False) )
        if span != m_num:
            log.warning("[{gabid}] {species}.{chrom}:{start}-{end}.".format(**instance._asdict()) + "(span) %d != %d (matches)" % (span, m_num))
            return None
        return instance
183793
def to_phonetics(self):
        """"""
        tr = Transcriber()
        self.transcribed_phonetics = [tr.transcribe(line) for line in self.text]
791545
def short_encode(input, errors='strict'):
    
    if not isinstance(input, text_type):
        input = text_type(input, sys.getdefaultencoding(), errors)
    length = len(input)
    input = unicodedata.normalize('NFKC', input)
    return input.translate(short_table), length
368673
def sequence(self, per_exon=False):
        
        db = self.db
        if not per_exon:
            start = self.txStart + 1
            return _sequence(db, self.chrom, start, self.txEnd)
        else:
            # TODO: use same strategy as cds_sequence to reduce # of requests.
            seqs = []
            for start, end in self.exons:
                seqs.append(_sequence(db, self.chrom, start + 1, end))
            return seqs
140952
def _create_sentence_objects(self):
        """"""
        sentence_objects = []
        sentences = sent_tokenize(self.raw, tokenizer=self.tokenizer)
        char_index = 0  # Keeps track of character index within the blob
        for sent in sentences:
            # Compute the start and end indices of the sentence
            # within the blob. This only works if the sentence splitter
            # does not perform any character replacements or changes to
            # white space.
            # Working: NLTKPunktTokenizer
            # Not working: PatternTokenizer
            try:
                start_index = self.raw.index(sent, char_index)
                char_index += len(sent)
                end_index = start_index + len(sent)
            except ValueError:
                start_index = None
                end_index = None
            # Sentences share the same models as their parent blob
            s = Sentence(
                sent,
                start_index=start_index,
                end_index=end_index,
                tokenizer=self.tokenizer,
                np_extractor=self.np_extractor,
                pos_tagger=self.pos_tagger,
                analyzer=self.analyzer,
                parser=self.parser,
                classifier=self.classifier)
            sentence_objects.append(s)
        return sentence_objects
519704
def generate_subtitles( # pylint: disable=too-many-locals,too-many-arguments
        source_path,
        output=None,
        concurrency=DEFAULT_CONCURRENCY,
        src_language=DEFAULT_SRC_LANGUAGE,
        dst_language=DEFAULT_DST_LANGUAGE,
        subtitle_file_format=DEFAULT_SUBTITLE_FORMAT,
        api_key=None,
    ):
    
    audio_filename, audio_rate = extract_audio(source_path)

    regions = find_speech_regions(audio_filename)

    pool = multiprocessing.Pool(concurrency)
    converter = FLACConverter(source_path=audio_filename)
    recognizer = SpeechRecognizer(language=src_language, rate=audio_rate,
                                  api_key=GOOGLE_SPEECH_API_KEY)

    transcripts = []
    if regions:
        try:
            widgets = ["Converting speech regions to FLAC files: ", Percentage(), ' ', Bar(), ' ',
                       ETA()]
            pbar = ProgressBar(widgets=widgets, maxval=len(regions)).start()
            extracted_regions = []
            for i, extracted_region in enumerate(pool.imap(converter, regions)):
                extracted_regions.append(extracted_region)
                pbar.update(i)
            pbar.finish()

            widgets = ["Performing speech recognition: ", Percentage(), ' ', Bar(), ' ', ETA()]
            pbar = ProgressBar(widgets=widgets, maxval=len(regions)).start()

            for i, transcript in enumerate(pool.imap(recognizer, extracted_regions)):
                transcripts.append(transcript)
                pbar.update(i)
            pbar.finish()

            if src_language.split("-")[0] != dst_language.split("-")[0]:
                if api_key:
                    google_translate_api_key = api_key
                    translator = Translator(dst_language, google_translate_api_key,
                                            dst=dst_language,
                                            src=src_language)
                    prompt = "Translating from {0} to {1}: ".format(src_language, dst_language)
                    widgets = [prompt, Percentage(), ' ', Bar(), ' ', ETA()]
                    pbar = ProgressBar(widgets=widgets, maxval=len(regions)).start()
                    translated_transcripts = []
                    for i, transcript in enumerate(pool.imap(translator, transcripts)):
                        translated_transcripts.append(transcript)
                        pbar.update(i)
                    pbar.finish()
                    transcripts = translated_transcripts
                else:
                    print(
                        "Error: Subtitle translation requires specified Google Translate API key. "
                        "See --help for further information."
                    )
                    return 1

        except KeyboardInterrupt:
            pbar.finish()
            pool.terminate()
            pool.join()
            print("Cancelling transcription")
            raise

    timed_subtitles = [(r, t) for r, t in zip(regions, transcripts) if t]
    formatter = FORMATTERS.get(subtitle_file_format)
    formatted_subtitles = formatter(timed_subtitles)

    dest = output

    if not dest:
        base = os.path.splitext(source_path)[0]
        dest = "{base}.{format}".format(base=base, format=subtitle_file_format)

    with open(dest, 'wb') as output_file:
        output_file.write(formatted_subtitles.encode("utf-8"))

    os.remove(audio_filename)

    return dest
1237049
def crude_tokenizer(line):
    """"""
    tokens = []
    buffer = ''
    for c in line.strip():
        if c == ' ' or c in string.punctuation:
            if buffer:
                tokens.append(buffer)
                buffer = ''
        else:
            buffer += c
    if buffer: tokens.append(buffer)
    return tokens
1096340
def translate_text(estimator, subtokenizer, txt):
  """"""
  encoded_txt = _encode_and_add_eos(txt, subtokenizer)

  def input_fn():
    ds = tf.data.Dataset.from_tensors(encoded_txt)
    ds = ds.batch(_DECODE_BATCH_SIZE)
    return ds

  predictions = estimator.predict(input_fn)
  translation = next(predictions)["outputs"]
  translation = _trim_and_decode(translation, subtokenizer)
  print("Translation of \"%s\": \"%s\"" % (txt, translation))