Use the work-breaking option.
        TODO: Maybe incorporate this into ``convert_corpus()``
        TODO: Write test for this
351585
def divide_works(self, corpus):
        
        if corpus == 'tlg':
            orig_dir_rel = '~/cltk_data/originals/tlg'
            works_dir_rel = '~/cltk_data/greek/text/tlg/individual_works'
            file_prefix = 'TLG'
            latin = False
        elif corpus == 'phi5':
            orig_dir_rel = '~/cltk_data/originals/phi5'
            works_dir_rel = '~/cltk_data/latin/text/phi5/individual_works'
            file_prefix = 'LAT'
            latin = True  # this is for the optional TLGU argument to convert()

        orig_dir = os.path.expanduser(orig_dir_rel)
        works_dir = os.path.expanduser(works_dir_rel)
        if not os.path.exists(works_dir):
            os.makedirs(works_dir)

        files = os.listdir(orig_dir)
        texts = [x for x in files if x.endswith('.TXT') and x.startswith(file_prefix)]

        for file in texts:
            orig_file_path = os.path.join(orig_dir, file)
            new_file_path = os.path.join(works_dir, file)

            try:
                self.convert(orig_file_path, new_file_path, divide_works=True, latin=latin)
                logger.info('Writing files at %s to %s.', orig_file_path, works_dir)
            except Exception as err:
                logger.error('Failed to convert files: %s.', err)
399936
def convert(self, values, nan_rep, encoding, errors):
        

        self.values = Int64Index(np.arange(self.table.nrows))
        return self
34604
def generate_corpus(self, text):
        
        if isinstance(text, str):
            sentences = self.sentence_split(text)
        else:
            sentences = []
            for line in text:
                sentences += self.sentence_split(line)
        passing = filter(self.test_sentence_input, sentences)
        runs = map(self.word_split, passing)
        return runs
904623
def _transliterate (self, text, outFormat):
        
        result = []
        text = self._preprocess(text)
        i = 0
        while i < len(text):
            if text[i].isspace(): 
                result.append(text[i])
                i = i+1
            else: 
                chr = self._getNextChar(text, i)
                try:
                    result.append(self[chr].unichr)
                except KeyError:
                    result.append(_unrecognised(chr))
                i = i + len(chr)
        return result
775797
def _export_corpus(self):
        
        # bin/mallet import-file --input /Users/erickpeirson/mycorpus_docs.txt
        #     --output mytopic-input.mallet --keep-sequence --remove-stopwords

        if not os.path.exists(self.mallet_bin):
            raise IOError("MALLET path invalid or non-existent.")
        self.input_path = os.path.join(self.temp, "input.mallet")

        exit = subprocess.call([
                self.mallet_bin,
                'import-file',
                '--input', self.corpus_path,
                '--output', self.input_path,
                '--keep-sequence',          # Required for LDA.
                '--remove-stopwords'])      # Probably redundant.

        if exit != 0:
            msg = "MALLET import-file failed with exit code {0}.".format(exit)
            raise RuntimeError(msg)
371595
def main(lang="en", output_dir=None, n_iter=25):
    
    nlp = spacy.blank(lang)
    # add the tagger to the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    tagger = nlp.create_pipe("tagger")
    # Add the tags. This needs to be done before you start training.
    for tag, values in TAG_MAP.items():
        tagger.add_label(tag, values)
    nlp.add_pipe(tagger)

    optimizer = nlp.begin_training()
    for i in range(n_iter):
        random.shuffle(TRAIN_DATA)
        losses = {}
        # batch up the examples using spaCy's minibatch
        batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
        for batch in batches:
            texts, annotations = zip(*batch)
            nlp.update(texts, annotations, sgd=optimizer, losses=losses)
        print("Losses", losses)

    # test the trained model
    test_text = "I like blue eggs"
    doc = nlp(test_text)
    print("Tags", [(t.text, t.tag_, t.pos_) for t in doc])

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the save model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        doc = nlp2(test_text)
        print("Tags", [(t.text, t.tag_, t.pos_) for t in doc])
896482
def translate(self, text):
        

        # Reset substitution counter
        self.count = 0

        # Process text
        return self._make_regex().sub(self, text)
798051
def translify(text):
    """"""
    try:
        res = translit.translify(smart_text(text, encoding))
    except Exception as err:
        # because filter must die silently
        res = default_value % {'error': err, 'value': text}
    return res
355674
def _insert_breaklines(self):
    
    target_chunks = ChunkList()
    for chunk in self:
      if chunk.word[-1] == ' ' and chunk.has_cjk():
        chunk.word = chunk.word[:-1]
        target_chunks.append(chunk)
        target_chunks.append(chunk.breakline())
      else:
        target_chunks.append(chunk)
    self.list = target_chunks
1183535
def _convert_to_spans(self, tree, start, set, parent = None): 
        "" 
        if len(tree) == 3:
            # Binary rule.
            # Remove unary collapsing.
            current = self._remove_vertical_markovization(tree[0]).split("+")
            split = self._convert_to_spans(tree[1], start, set, None)
            end = self._convert_to_spans(tree[2], split + 1, set, current[-1])
            
            # Add phrases to set
            if current[0] != parent: 
                set.add((current[0], start, end))
            for nt in current[1:]:
                set.add((nt, start, end))
            return end
        
        elif len(tree) == 2:
            # Unary rule.
            
            # Can have a constituent if it is collapsed.
            current = self._remove_vertical_markovization(tree[0]).split("+")
            for nt in current[:-1]:
                set.add((nt, start, start))
            return start
387479
def process_corpus(self):
		
		for doc in self.corpus_list:
			doc = wt(doc)
			sentence = []
			for word in doc:
				if word not in self.stop_words and word not in self.punctuation:
					word = self.stemmer.stem(word)
					sentence.append(word)
			self.processed_corpus.append(sentence)
727452
def detranslify(text):
    """"""
    try:
        res = translit.detranslify(text)
    except Exception as err:
        # because filter must die silently
        res = default_value % {'error': err, 'value': text}
    return res
694045
def _load_corpus(name, data_home=None):
    
    info = DATASETS[name]
    return Corpus(name, data_home=data_home, **info)
614317
def transform_returns(self, raw_lines, tre_return_grammar=None, use_mock=None, is_async=False):
        """"""
        lines = []  # transformed lines
        tco = False  # whether tco was done
        tre = False  # whether tre was done
        level = 0  # indentation level
        disabled_until_level = None  # whether inside of a disabled block
        attempt_tre = tre_return_grammar is not None  # whether to even attempt tre
        attempt_tco = not is_async and not self.no_tco  # whether to even attempt tco

        if is_async:
            internal_assert(not attempt_tre and not attempt_tco, "cannot tail call optimize async functions")

        for line in raw_lines:
            indent, body, dedent = split_leading_trailing_indent(line)
            base, comment = split_comment(body)

            level += ind_change(indent)

            if disabled_until_level is not None:
                if level <= disabled_until_level:
                    disabled_until_level = None

            if disabled_until_level is None:

                # tco and tre don't support generators
                if not is_async and self.yield_regex.search(body):
                    lines = raw_lines  # reset lines
                    break

                # don't touch inner functions
                elif self.def_regex.match(body):
                    disabled_until_level = level

                # tco and tre shouldn't touch scopes that depend on actual return statements
                #  or scopes where we can't insert a continue
                elif not is_async and self.tre_disable_regex.match(body):
                    disabled_until_level = level

                else:

                    if is_async:
                        if self.return_regex.match(base):
                            to_return = base[len("return"):].strip()
                            if to_return:  # leave empty return statements alone
                                line = indent + "raise _coconut.asyncio.Return(" + to_return + ")" + comment + dedent

                    tre_base = None
                    if attempt_tre:
                        with self.complain_on_err():
                            tre_base = transform(tre_return_grammar, base)
                        if tre_base is not None:
                            line = indent + tre_base + comment + dedent
                            tre = True
                            # when tco is available, tre falls back on it if the function is changed
                            tco = not self.no_tco

                    if attempt_tco and tre_base is None:  # don't attempt tco if tre succeeded
                        tco_base = None
                        with self.complain_on_err():
                            tco_base = transform(self.tco_return, base)
                        if tco_base is not None:
                            line = indent + tco_base + comment + dedent
                            tco = True

            level += ind_change(dedent)
            lines.append(line)

        func_code = "".join(lines)
        if is_async:
            return func_code
        else:
            return func_code, tco, tre
1065191
def tokenizer(text):
    

    stream = deque(text.split("\n"))

    while len(stream) > 0:
        line = stream.popleft()

        if line.startswith("#METABOLOMICS WORKBENCH"):
            yield KeyValue("#METABOLOMICS WORKBENCH", "\n")
            yield KeyValue("HEADER", line)

            for identifier in line.split(" "):
                if ":" in identifier:
                    key, value = identifier.split(":")
                    yield KeyValue(key, value)

        elif line.startswith("#ANALYSIS TYPE"):
            yield KeyValue("HEADER", line)

        elif line.startswith("#SUBJECT_SAMPLE_FACTORS:"):
            yield KeyValue("#ENDSECTION", "\n")
            yield KeyValue("#SUBJECT_SAMPLE_FACTORS", "\n")

        elif line.startswith("#"):
            yield KeyValue("#ENDSECTION", "\n")
            yield KeyValue(line.strip(), "\n")

        elif line.startswith("SUBJECT_SAMPLE_FACTORS"):
            key, subject_type, local_sample_id, factors, additional_sample_data = line.split("\t")
            # factors = [dict([[i.strip() for i in f.split(":")]]) for f in factors.split("|")]
            yield SubjectSampleFactors(key.strip(), subject_type, local_sample_id, factors, additional_sample_data)

        elif line.endswith("_START"):
            yield KeyValue(line, "\n")

            while not line.endswith("_END"):
                line = stream.popleft()
                if line.endswith("_END"):
                    yield KeyValue(line.strip(), "\n")
                else:
                    data = line.split("\t")
                    yield KeyValue(data[0], tuple(data))

        else:
            if line:
                if line.startswith("MS:MS_RESULTS_FILE") or line.startswith("NM:NMR_RESULTS_FILE"):
                    try:
                        key, value, extra = line.split("\t")
                        extra_key, extra_value = extra.strip().split(":")
                        yield KeyValueExtra(key.strip()[3:], value, extra_key, extra_value)
                    except ValueError:
                        key, value = line.split("\t")
                        yield KeyValue(key.strip()[3:], value)
                else:
                    try:
                        key, value = line.split("\t")
                        if ":" in key:
                            if key.startswith("MS_METABOLITE_DATA:UNITS"):
                                yield KeyValue(key.strip(), value)
                            else:
                                yield KeyValue(key.strip()[3:], value)
                        else:
                            yield KeyValue(key.strip(), value)
                    except ValueError:
                        print("LINE WITH ERROR:\n\t", repr(line))
                        raise

    yield KeyValue("#ENDSECTION", "\n")
    yield KeyValue("!#ENDFILE", "\n")
971578
def raw_corpus_chrf(hypotheses: Iterable[str], references: Iterable[str]) -> float:
    
    return sacrebleu.corpus_chrf(hypotheses, references, order=sacrebleu.CHRF_ORDER, beta=sacrebleu.CHRF_BETA,
                                 remove_whitespace=True)
622914
def collate(*iterables, **kwargs):
    
    key = kwargs.pop('key', lambda a: a)
    reverse = kwargs.pop('reverse', False)
    min_or_max = max if reverse else min

    rows = [iter(iterable) for iterable in iterables if iterable]
    next_values = {}
    by_key = []

    def gather_next_value(row, index):
        try:
            next_value = next(row)
        except StopIteration:
            pass
        else:
            next_values[index] = next_value
            by_key.append((key(next_value), index))

    for index, row in enumerate(rows):
        gather_next_value(row, index)

    while by_key:
        key_value, index = min_or_max(by_key)
        by_key.remove((key_value, index))
        next_value = next_values.pop(index)
        yield next_value
        gather_next_value(rows[index], index)