Generate a vocabulary from txt files with example-per-line.
764853
def get_or_generate_txt_vocab(data_dir, vocab_filename, vocab_size,
                              filepatterns):
  """"""
  if isinstance(filepatterns, str):
    filepatterns = [filepatterns]

  def generate():
    tf.logging.info("Generating vocab from %s", filepatterns)
    for filepattern in filepatterns:
      for filename in tf.gfile.Glob(filepattern):
        with tf.gfile.GFile(filename, mode="r") as source_file:
          for line in source_file:
            yield line.strip()

  return get_or_generate_vocab_inner(data_dir, vocab_filename, vocab_size,
                                     generate())
741783
def from_text(cls, text, mapping='mapping'):
        
        graphemes = Counter(grapheme_pattern.findall(text))
        specs = [
            OrderedDict([
                (cls.GRAPHEME_COL, grapheme),
                ('frequency', frequency),
                (mapping, grapheme)])
            for grapheme, frequency in graphemes.most_common()]
        return cls(*specs)
443201
def build_input_data(sentences, labels, vocabulary):
    """"""
    x = np.array([[vocabulary[word] for word in sentence] for sentence in sentences])
    y = np.array(labels)
    return [x, y]
540787
def make_data_splits(label_type, train_rec_type="text_and_wordlist", max_samples=1000,
                     seed=0, tgt_dir=TGT_DIR):
    

    feat_dir = os.path.join(tgt_dir, "feat")
    test_prefix_fn=os.path.join(tgt_dir, "test_prefixes.txt")
    valid_prefix_fn=os.path.join(tgt_dir, "valid_prefixes.txt")
    with open(test_prefix_fn) as f:
        prefixes = f.readlines()
        test_prefixes = [("TEXT/" + prefix).strip() for prefix in prefixes]
    with open(valid_prefix_fn) as f:
        prefixes = f.readlines()
        valid_prefixes = [("TEXT/" + prefix).strip() for prefix in prefixes]

    label_dir = os.path.join(tgt_dir, "label")
    prefixes = get_story_prefixes(label_type, label_dir=label_dir)
    prefixes = list(set(prefixes) - set(valid_prefixes))
    prefixes = list(set(prefixes) - set(test_prefixes))
    prefixes = utils.filter_by_size(
        feat_dir, prefixes, "fbank", max_samples)

    if train_rec_type == "text":
        train_prefixes = prefixes
    else:
        wordlist_prefixes = [prefix for prefix in os.listdir(os.path.join(label_dir, "WORDLIST"))
                             if prefix.endswith("phonemes")]
        wordlist_prefixes = [os.path.splitext(os.path.join("WORDLIST", prefix))[0]
                             for prefix in wordlist_prefixes]
        wordlist_prefixes = utils.filter_by_size(
                feat_dir, wordlist_prefixes, "fbank", max_samples)
        if train_rec_type == "wordlist":
            prefixes = wordlist_prefixes
        elif train_rec_type == "text_and_wordlist":
            prefixes.extend(wordlist_prefixes)
        else:
            raise PersephoneException("train_rec_type='%s' not supported." % train_rec_type)
        train_prefixes = prefixes
    random.seed(seed)
    random.shuffle(train_prefixes)

    return train_prefixes, valid_prefixes, test_prefixes
564879
def load_from_file(cls, filename_prefix):
    """"""
    filename = cls._filename(filename_prefix)
    lines, _ = cls._read_lines_from_file(filename)
    # Strip wrapping single quotes
    vocab_list = [line[1:-1] for line in lines]
    return cls(vocab_list=vocab_list)
956916
def tabbed_parsing_token_generator(data_dir, tmp_dir, train, prefix,
                                   source_vocab_size, target_vocab_size):
  """"""
  filename = "parsing_{0}.pairs".format("train" if train else "dev")
  source_vocab = generator_utils.get_or_generate_tabbed_vocab(
      data_dir, tmp_dir, filename, 0,
      prefix + "_source.tokens.vocab.%d" % source_vocab_size, source_vocab_size)
  target_vocab = generator_utils.get_or_generate_tabbed_vocab(
      data_dir, tmp_dir, filename, 1,
      prefix + "_target.tokens.vocab.%d" % target_vocab_size, target_vocab_size)
  pair_filepath = os.path.join(tmp_dir, filename)
  return text_problems.text2text_generate_encoded(
      text_problems.text2text_txt_tab_iterator(pair_filepath), source_vocab,
      target_vocab)
951042
def gen_textfiles_from_filenames(
        filenames: Iterable[str]) -> Generator[TextIO, None, None]:
    
    for filename in filenames:
        with open(filename) as f:
            yield f
559850
def construct_txt_file(self):
        """"""
        textlines = ['Prediction of noncovalent interactions for PDB structure %s' % self.mol.pymol_name.upper(), ]
        textlines.append("=" * len(textlines[0]))
        textlines.append('Created on %s using PLIP v%s\n' % (time.strftime("%Y/%m/%d"), __version__))
        textlines.append('If you are using PLIP in your work, please cite:')
        textlines.append('Salentin,S. et al. PLIP: fully automated protein-ligand interaction profiler.')
        textlines.append('Nucl. Acids Res. (1 July 2015) 43 (W1): W443-W447. doi: 10.1093/nar/gkv315\n')
        if len(self.excluded) != 0:
            textlines.append('Excluded molecules as ligands: %s\n' % ','.join([lig for lig in self.excluded]))
        if config.DNARECEPTOR:
            textlines.append('DNA/RNA in structure was chosen as the receptor part.\n')
        return textlines
1144238
def read_examples(input_file):
    """"""
    examples = []
    unique_id = 0
    with open(input_file, "r", encoding='utf-8') as reader:
        while True:
            line = reader.readline()
            if not line:
                break
            line = line.strip()
            text_a = None
            text_b = None
            m = re.match(r"^(.*) \|\|\| (.*)$", line)
            if m is None:
                text_a = line
            else:
                text_a = m.group(1)
                text_b = m.group(2)
            examples.append(
                InputExample(unique_id=unique_id, text_a=text_a, text_b=text_b))
            unique_id += 1
    return examples
1035823
def _example_generator(filename):
  
  for idx, line in enumerate(tf.gfile.Open(filename, "rb")):
    if idx == 0: continue  # skip header
    line = text_encoder.to_unicode_utf8(line.strip())
    split_line = line.split("\t")
    # Works for both splits even though dev has some extra human labels.
    yield {
        "premise": split_line[8],
        "hypothesis": split_line[9],
        "label": split_line[-1]
    }
1065191
def tokenizer(text):
    

    stream = deque(text.split("\n"))

    while len(stream) > 0:
        line = stream.popleft()

        if line.startswith("#METABOLOMICS WORKBENCH"):
            yield KeyValue("#METABOLOMICS WORKBENCH", "\n")
            yield KeyValue("HEADER", line)

            for identifier in line.split(" "):
                if ":" in identifier:
                    key, value = identifier.split(":")
                    yield KeyValue(key, value)

        elif line.startswith("#ANALYSIS TYPE"):
            yield KeyValue("HEADER", line)

        elif line.startswith("#SUBJECT_SAMPLE_FACTORS:"):
            yield KeyValue("#ENDSECTION", "\n")
            yield KeyValue("#SUBJECT_SAMPLE_FACTORS", "\n")

        elif line.startswith("#"):
            yield KeyValue("#ENDSECTION", "\n")
            yield KeyValue(line.strip(), "\n")

        elif line.startswith("SUBJECT_SAMPLE_FACTORS"):
            key, subject_type, local_sample_id, factors, additional_sample_data = line.split("\t")
            # factors = [dict([[i.strip() for i in f.split(":")]]) for f in factors.split("|")]
            yield SubjectSampleFactors(key.strip(), subject_type, local_sample_id, factors, additional_sample_data)

        elif line.endswith("_START"):
            yield KeyValue(line, "\n")

            while not line.endswith("_END"):
                line = stream.popleft()
                if line.endswith("_END"):
                    yield KeyValue(line.strip(), "\n")
                else:
                    data = line.split("\t")
                    yield KeyValue(data[0], tuple(data))

        else:
            if line:
                if line.startswith("MS:MS_RESULTS_FILE") or line.startswith("NM:NMR_RESULTS_FILE"):
                    try:
                        key, value, extra = line.split("\t")
                        extra_key, extra_value = extra.strip().split(":")
                        yield KeyValueExtra(key.strip()[3:], value, extra_key, extra_value)
                    except ValueError:
                        key, value = line.split("\t")
                        yield KeyValue(key.strip()[3:], value)
                else:
                    try:
                        key, value = line.split("\t")
                        if ":" in key:
                            if key.startswith("MS_METABOLITE_DATA:UNITS"):
                                yield KeyValue(key.strip(), value)
                            else:
                                yield KeyValue(key.strip()[3:], value)
                        else:
                            yield KeyValue(key.strip(), value)
                    except ValueError:
                        print("LINE WITH ERROR:\n\t", repr(line))
                        raise

    yield KeyValue("#ENDSECTION", "\n")
    yield KeyValue("!#ENDFILE", "\n")
586755
def parse_text_to_dict(self, txt):
        
        op = {}
        print('TODO - import NLP, split into verbs / nouns')
        op['nouns'] = txt
        op['verbs'] = txt
        
        return op
58401
def _build_vocab(filename, vocab_dir, vocab_name):
  
  vocab_path = os.path.join(vocab_dir, vocab_name)
  if not tf.gfile.Exists(vocab_path):
    with tf.gfile.GFile(filename, "r") as f:
      data = f.read().split()
    counter = collections.Counter(data)
    count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
    words, _ = list(zip(*count_pairs))
    encoder = text_encoder.TokenTextEncoder(None, vocab_list=words)
    encoder.store_to_file(vocab_path)
  else:
    encoder = text_encoder.TokenTextEncoder(vocab_path)
  return encoder
587146
def get_vocabulary(preprocess_output_dir, name):
  
  vocab_file = os.path.join(preprocess_output_dir, CATEGORICAL_ANALYSIS % name)
  if not file_io.file_exists(vocab_file):
    raise ValueError('File %s not found in %s' %
                     (CATEGORICAL_ANALYSIS % name, preprocess_output_dir))

  labels = python_portable_string(
      file_io.read_file_to_string(vocab_file)).split('\n')
  label_values = [x for x in labels if x]  # remove empty lines

  return label_values
703954
def phrases_map(data):
  """"""
  (entry, text_fn) = data
  text = text_fn()
  filename = entry.filename

  logging.debug("Got %s", filename)
  for s in split_into_sentences(text):
    words = split_into_words(s.lower())
    if len(words) < PHRASE_LENGTH:
      yield (":".join(words), filename)
      continue
    for i in range(0, len(words) - PHRASE_LENGTH):
      yield (":".join(words[i:i+PHRASE_LENGTH]), filename)
1125439
def _generate_examples(self, file_path):
    

    with tf.io.gfile.GFile(file_path) as f:
      raw_data = csv.DictReader(f)
      for row in raw_data:
        survive_val = row.pop("survived")
        yield {
            "survived": convert_to_label(survive_val, _SURVIVED_DICT),
            "features": {
                name: FEATURE_DICT[name][1](value)
                for name, value in row.items()
            }
        }
787913
def create_training_instances(x):
    """"""
    (input_files, out, tokenizer, max_seq_length, dupe_factor,
     short_seq_prob, masked_lm_prob, max_predictions_per_seq, rng) = x
    time_start = time.time()
    logging.info('Processing %s', input_files)
    all_documents = [[]]

    # Input file format:
    # (1) One sentence per line. These should ideally be actual sentences, not
    # entire paragraphs or arbitrary spans of text. (Because we use the
    # sentence boundaries for the "next sentence prediction" task).
    # (2) Blank lines between documents. Document boundaries are needed so
    # that the "next sentence prediction" task doesn't span between documents.
    for input_file in input_files:
        with io.open(input_file, 'r', encoding='UTF-8') as reader:
            while True:
                line = reader.readline()
                if not line:
                    break
                line = line.strip()

                # Empty lines are used as document delimiters
                if not line:
                    all_documents.append([])
                tokens = tokenizer(line) if not args.tokenized else line.split(' ')
                if tokens:
                    all_documents[-1].append(tokens)

    # Remove empty documents
    all_documents = [x for x in all_documents if x]
    rng.shuffle(all_documents)

    vocab_words = tokenizer.vocab.idx_to_token
    instances = []
    for _ in range(dupe_factor):
        for document_index in range(len(all_documents)):
            instances.extend(
                create_instances_from_document(
                    all_documents, document_index, max_seq_length,
                    short_seq_prob, masked_lm_prob, max_predictions_per_seq,
                    vocab_words, rng))

    rng.shuffle(instances)
    input_ids = []
    segment_ids = []
    masked_lm_positions = []
    masked_lm_ids = []
    masked_lm_weights = []
    next_sentence_labels = []
    valid_lengths = []

    for inst_index, instance in enumerate(instances):
        feature = transform(instance, tokenizer, max_seq_length, max_predictions_per_seq, False)
        input_ids.append(
            np.ascontiguousarray(feature['input_ids'], dtype='int32'))
        segment_ids.append(
            np.ascontiguousarray(feature['segment_ids'], dtype='int32'))
        masked_lm_positions.append(
            np.ascontiguousarray(feature['masked_lm_positions'], dtype='int32'))
        masked_lm_ids.append(np.ascontiguousarray(feature['masked_lm_ids'], dtype='int32'))
        masked_lm_weights.append(
            np.ascontiguousarray(feature['masked_lm_weights'], dtype='float32'))
        next_sentence_labels.append(feature['next_sentence_labels'][0])
        valid_lengths.append(feature['valid_lengths'][0])
        if inst_index < 20:
            print_example(instance, feature)

    features = (input_ids, segment_ids, masked_lm_positions, masked_lm_ids, \
                masked_lm_weights, next_sentence_labels, valid_lengths)

    logging.info('*** Writing to output file %s ***', out)
    if args.format == 'numpy':
        write_to_files_np(features, tokenizer, args.max_seq_length,
                          args.max_predictions_per_seq, [out])
    elif args.format == 'recordio':
        write_to_files_rec(instances, tokenizer, args.max_seq_length,
                           args.max_predictions_per_seq, [out])
    else:
        raise ValueError('unsupported format: %s'%args.format)

    time_end = time.time()
    logging.info('Process %d files took %.1f s', len(input_files), time_end - time_start)