a helper method for multi-processing file transcription
340819
def _transcribe_files(self, file_list, file_mimetype):

        '''  '''

    # import dependencies
        import queue
        from threading import Thread

    # define multithreading function
        def _recognize_file(file_path, file_mimetype, queue):

            details = {
                'error': '',
                'results': []
            }

            file_data = open(file_path, 'rb')
            try:
                transcript = self.client.recognize(file_data, file_mimetype, continuous=True)
                if transcript['results']:
                    for result in transcript['results']:
                        details['results'].append(result['alternatives'][0])
            except Exception as err:
                details['error'] = err

            queue.put(details)

    # construct queue
        q = queue.Queue()

    # send clips to watson for transcription in separate threads
        threads = []
        for file in file_list:
            watson_job = Thread(target=_recognize_file, args=(file, file_mimetype, q))
            watson_job.start()
            threads.append(watson_job)

    # wait for threads to end
        for t in threads:
            t.join()

    # construct result
        transcript_details = {
            'error': '',
            'segments': []
        }
        for i in range(len(file_list)):
            job_details = q.get()
            if job_details['error']:
                transcript_details['error'] = job_details['error']
            transcript_details['segments'].extend(job_details['results'])

        return transcript_details
648249
def build_dataset(filename, max_lines=-1):
    """"""
    encodings = dict(list(map(reversed, enumerate(string.printable))))
    digitize = lambda char: encodings[char] if char in encodings else len(encodings)
    encode_line = lambda line: np.array(list(map(digitize, line)))
    nonblank_line = lambda line: len(line) > 2

    with open(filename) as f:
        lines = f.readlines()

    encoded_lines = list(map(encode_line, list(filter(nonblank_line, lines))[:max_lines]))
    num_outputs = len(encodings) + 1

    return encoded_lines, num_outputs
544274
def pre_process_data(filepath):
    
    positive_path = os.path.join(filepath, 'pos')
    negative_path = os.path.join(filepath, 'neg')

    pos_label = 1
    neg_label = 0

    dataset = []

    for filename in glob.glob(os.path.join(positive_path, '*.txt')):
        with open(filename, 'r') as f:
            dataset.append((pos_label, f.read()))

    for filename in glob.glob(os.path.join(negative_path, '*.txt')):
        with open(filename, 'r') as f:
            dataset.append((neg_label, f.read()))

    shuffle(dataset)

    return dataset
1091562
def gen_docs(corpus, lemmatize, rm_stops):
    

    assert corpus in ['phi5', 'tlg']

    if corpus == 'phi5':
        language = 'latin'
        filepaths = assemble_phi5_author_filepaths()
        jv_replacer = JVReplacer()
        text_cleaner = phi5_plaintext_cleanup
        word_tokenizer = nltk_tokenize_words
        if rm_stops:
            stops = latin_stops
        else:
            stops = None
    elif corpus == 'tlg':
        language = 'greek'
        filepaths = assemble_tlg_author_filepaths()
        text_cleaner = tlg_plaintext_cleanup
        word_tokenizer = nltk_tokenize_words

        if rm_stops:
            stops = latin_stops
        else:
            stops = None

    if lemmatize:
        lemmatizer = LemmaReplacer(language)

    sent_tokenizer = TokenizeSentence(language)

    for filepath in filepaths:
        with open(filepath) as f:
            text = f.read()
        # light first-pass cleanup, before sentence tokenization (which relies on punctuation)
        text = text_cleaner(text, rm_punctuation=False, rm_periods=False)
        sent_tokens = sent_tokenizer.tokenize_sentences(text)
        # doc_sentences = []
        for sentence in sent_tokens:
            # a second cleanup at sentence-level, to rm all punctuation
            sentence = text_cleaner(sentence, rm_punctuation=True, rm_periods=True)
            sentence = word_tokenizer(sentence)
            sentence = [s.lower() for s in sentence]
            sentence = [w for w in sentence if w]
            if language == 'latin':
                sentence = [w[1:] if w.startswith('-') else w for w in sentence]

            if stops:
                sentence = [w for w in sentence if w not in stops]

            sentence = [w for w in sentence if len(w) > 1]  # rm short words

            if sentence:
                sentence = sentence

            if lemmatize:
                sentence = lemmatizer.lemmatize(sentence)
            if sentence and language == 'latin':
                sentence = [jv_replacer.replace(word) for word in sentence]
            if sentence:
                yield sentence
1038775
def extract_rar (archive, compression, cmd, verbosity, interactive, outdir):
    """"""
    cmdlist = [cmd, 'x']
    if not interactive:
        cmdlist.extend(['-p-', '-y'])
    cmdlist.extend(['--', os.path.abspath(archive)])
    return (cmdlist, {'cwd': outdir})
956916
def tabbed_parsing_token_generator(data_dir, tmp_dir, train, prefix,
                                   source_vocab_size, target_vocab_size):
  """"""
  filename = "parsing_{0}.pairs".format("train" if train else "dev")
  source_vocab = generator_utils.get_or_generate_tabbed_vocab(
      data_dir, tmp_dir, filename, 0,
      prefix + "_source.tokens.vocab.%d" % source_vocab_size, source_vocab_size)
  target_vocab = generator_utils.get_or_generate_tabbed_vocab(
      data_dir, tmp_dir, filename, 1,
      prefix + "_target.tokens.vocab.%d" % target_vocab_size, target_vocab_size)
  pair_filepath = os.path.join(tmp_dir, filename)
  return text_problems.text2text_generate_encoded(
      text_problems.text2text_txt_tab_iterator(pair_filepath), source_vocab,
      target_vocab)
166232
def reconstruct_files(input_dir):
    ''''''
    input_dir = input_dir.rstrip('/')
    with nl.notify('Attempting to organize/reconstruct directory'):
        # Some datasets start with a ".", which confuses many programs
        for r,ds,fs in os.walk(input_dir):
            for f in fs:
                if f[0]=='.':
                    shutil.move(os.path.join(r,f),os.path.join(r,'i'+f))
        nl.dicom.organize_dir(input_dir)
        output_dir = '%s-sorted' % input_dir
        if os.path.exists(output_dir):
            with nl.run_in(output_dir):
                for dset_dir in os.listdir('.'):
                    with nl.notify('creating dataset from %s' % dset_dir):
                        nl.dicom.create_dset(dset_dir)
        else:
            nl.notify('Warning: failed to auto-organize directory %s' % input_dir,level=nl.level.warning)
40880
def _encode_multipart(**kw):
    '  '
    boundary = '----------%s' % hex(int(time.time() * 1000))
    data = []
    for k, v in kw.iteritems():
        data.append('--%s' % boundary)
        if hasattr(v, 'read'):
            # file-like object:
            filename = getattr(v, 'name', '')
            content = v.read()
            data.append('Content-Disposition: form-data; name="%s"; filename="hidden"' % k)
            data.append('Content-Length: %d' % len(content))
            data.append('Content-Type: %s\r\n' % _guess_content_type(filename))
            data.append(content)
        else:
            data.append('Content-Disposition: form-data; name="%s"\r\n' % k)
            data.append(v.encode('utf-8') if isinstance(v, unicode) else v)
    data.append('--%s--\r\n' % boundary)
    return '\r\n'.join(data), boundary
1323759
def multicharacters(input)
      output = input.gsub(/\([Cc]\)/, "©")
      output = output.gsub(/\([Pp]\)/, "℗")
      output = output.gsub(/\([Rr]\)/, "®")
      output = output.gsub(/\((SM|sm|Sm)\)/, "℠")
      output = output.gsub(/\((TM|tm|Tm)\)/, "™")
      output = output.gsub(/\+-/, "±")
      output = output.gsub(/-\+/, "∓")
      output = output.gsub(/N[oO]\.?\s*(\d+)/, '№\1')
      output = output.gsub(/°C/, '℃')
      output = output.gsub(/°F/, '℉')
    end
700049
def get_input_sequence_files(data, default=None):
    
    if "files" not in data or data.get("files") is None:
        file1, file2 = None, None
    elif len(data["files"]) == 2:
        file1, file2 = data["files"]
    else:
        assert len(data["files"]) == 1, data["files"]
        file1, file2 = data["files"][0], None
    return file1, file2
1267574
function processFiles(filenames = [], options = defaultOptions) {

    if (options.convertUrls) {
        options.target = resolve(process.cwd(), options.convertUrls).split(PATH_SEP)
    }

    const uglies = []

    // process files
    filenames.forEach(filename => {
        try {
            const content = readFileSync(filename, 'utf8')
            if (content.length) {
                if (options.convertUrls) {
                    options.source = resolve(process.cwd(), filename).split(PATH_SEP)
                    options.source.pop()
                }
                uglies.push(processString(content, options))
            }
        } catch (e) {
            if (options.debug) {
                console.error(`uglifycss: unable to process "${filename}"\n${e.stack}`)
            } else {
                console.error(`uglifycss: unable to process "${filename}"\n\t${e}`)
            }
            process.exit(1)
        }
    })

    // return concat'd results
    return uglies.join('')
}
314623
def prep_fastq_inputs(in_files, data):
    
    if len(in_files) == 1 and _is_bam_input(in_files):
        out = _bgzip_from_bam(in_files[0], data["dirs"], data)
    elif len(in_files) == 1 and _is_cram_input(in_files):
        out = _bgzip_from_cram(in_files[0], data["dirs"], data)
    elif len(in_files) in [1, 2] and _ready_gzip_fastq(in_files, data):
        out = _symlink_in_files(in_files, data)
    else:
        if len(in_files) > 2:
            fpairs = fastq.combine_pairs(in_files)
            pair_types = set([len(xs) for xs in fpairs])
            assert len(pair_types) == 1
            fpairs.sort(key=lambda x: os.path.basename(x[0]))
            organized = [[xs[0] for xs in fpairs]]
            if len(fpairs[0]) > 1:
                organized.append([xs[1] for xs in fpairs])
            in_files = organized
        parallel = {"type": "local", "num_jobs": len(in_files),
                    "cores_per_job": max(1, data["config"]["algorithm"]["num_cores"] // len(in_files))}
        inputs = [{"in_file": x, "read_num": i, "dirs": data["dirs"], "config": data["config"],
                   "is_cwl": "cwl_keys" in data,
                   "rgnames": data["rgnames"]}
                  for i, x in enumerate(in_files) if x]
        out = run_multicore(_bgzip_from_fastq_parallel, [[d] for d in inputs], data["config"], parallel)
    return out
1368285
def main(_):
  """"""
  if FLAGS.subword_text_encoder_filename:
    encoder = text_encoder.SubwordTextEncoder(
        FLAGS.subword_text_encoder_filename)
  elif FLAGS.token_text_encoder_filename:
    encoder = text_encoder.TokenTextEncoder(FLAGS.token_text_encoder_filename)
  elif FLAGS.byte_text_encoder:
    encoder = text_encoder.ByteTextEncoder()
  else:
    encoder = None
  reader = tf.python_io.tf_record_iterator(FLAGS.input_filename)
  total_sequences = 0
  total_input_tokens = 0
  total_target_tokens = 0
  nonpadding_input_tokens = 0
  nonpadding_target_tokens = 0
  max_input_length = 0
  max_target_length = 0
  for record in reader:
    x = tf.train.Example()
    x.ParseFromString(record)
    inputs = [int(i) for i in x.features.feature["inputs"].int64_list.value]
    targets = [int(i) for i in x.features.feature["targets"].int64_list.value]
    if FLAGS.print_inputs:
      print("INPUTS:\n" + encoder.decode(inputs) if encoder else inputs)
    if FLAGS.print_targets:
      print("TARGETS:\n" + encoder.decode(targets) if encoder else targets)
    nonpadding_input_tokens += len(inputs) - inputs.count(0)
    nonpadding_target_tokens += len(targets) - targets.count(0)
    total_input_tokens += len(inputs)
    total_target_tokens += len(targets)
    total_sequences += 1
    max_input_length = max(max_input_length, len(inputs))
    max_target_length = max(max_target_length, len(targets))
    if FLAGS.print_all:
      for k, v in six.iteritems(x.features.feature):
        print("%s: %s" % (k, v.int64_list.value))

  print("total_sequences: %d" % total_sequences)
  print("total_input_tokens: %d" % total_input_tokens)
  print("total_target_tokens: %d" % total_target_tokens)
  print("nonpadding_input_tokens: %d" % nonpadding_input_tokens)
  print("nonpadding_target_tokens: %d" % nonpadding_target_tokens)
  print("max_input_length: %d" % max_input_length)
  print("max_target_length: %d" % max_target_length)
902714
def docs(self, fileids=None) -> Generator[str, str, None]:
        
        if not fileids:
            fileids = self.fileids()
        # Create a generator, loading one document into memory at a time.
        for path, encoding in self.abspaths(fileids, include_encoding=True):
            with codecs.open(path, 'r', encoding=encoding) as reader:
                if self.skip_keywords:
                    tmp_data = []
                    for line in reader:
                        skip = False
                        for keyword in self.skip_keywords:
                            if keyword in line:
                                skip = True
                        if not skip:
                            tmp_data.append(line)
                    yield ''.join(tmp_data)
                else:
                    yield reader.read()
1079643
def _batches_to_tmpfile(in_record_sep, in_column_sep, in_batches, batch_to_file_s):
        """"""
        for i, b2f in enumerate(batch_to_file_s):
            if b2f.is_tmpfile():
                input_str = BaseShellOperator._input_str(in_batches[i], in_record_sep, in_column_sep)
                b2f.write_tmpfile(input_str)
1245485
def concat_generator(filename, up_threshold, low_threshold=10):
  """"""
  txt = ""
  for line in tf.gfile.Open(filename):
    line = line.strip()
    if len(txt) + len(line) + 1 >= up_threshold:
      ret = txt
      txt = ""
      # We don't yield very short long parts to prevent noisy examples.
      if len(ret) > low_threshold and len(ret) < up_threshold:
        yield {"targets": ret}

    if not txt:
      txt = line
    else:
      txt = " ".join([txt, line])
126319
def process_inputs(self):
        
        ret = []
        files = self.args.files
        if files is None:
            return ret

        for fname in files:
            if fname == '-':
                if self.args.base64stdin:
                    for line in sys.stdin:
                        data = base64.b64decode(line)
                        ret.append(self.process_file(data, fname))

                    continue
                else:
                    fh = sys.stdin

            elif fname.endswith('.tar') or fname.endswith('.tar.gz'):
                sub = self.process_tar(fname)
                ret.append(sub)
                continue

            elif not os.path.isfile(fname):
                sub = self.process_dir(fname)
                ret.append(sub)
                continue

            else:
                fh = open(fname, 'rb')

            with fh:
                data = fh.read()
                sub = self.process_file(data, fname)
                ret.append(sub)

        return ret