Load a dialogue corpus; eventually, support pickles and potentially other formats
19396
def load_corpus(self, path, config):
        ''''''

        # use the default dataset if no path is provided
        # TODO -- change this to use a pre-saved dataset
        if path == '':
            path = self.default_path_to_corpus

        self.data = Corpus(path=path, config=self.data_config)
129992
def load(fin, dtype=np.float32, max_vocab=None):
    
    vocab = {}
    line = next(fin)
    data = line.strip().split(b' ')
    assert len(data) == 2
    words = int(data[0])
    if max_vocab is not None:
        words = min(max_vocab, words)
    size = int(data[1])
    arr = np.empty((words, size), dtype=dtype)
    i = 0
    for n_line, line in enumerate(fin):
        if i >= words:
            break
        token, v = _load_line(line, dtype, size)
        if token in vocab:
            parse_warn(b'Duplicated vocabulary ' + token)
            continue
        arr[i, :] = v
        vocab[token] = i
        i += 1
    if i != words:
        # Use + instead of formatting because python 3.4.* does not allow
        # format with bytes
        parse_warn(
            b'EOF before the defined size (read ' + bytes(i) + b', expected '
            + bytes(words) + b')'
        )
        arr = arr[:i, :]
    return arr, vocab
68548
def corpus(self):
        ''''''

        # Initialize the addcorpus subcommand's argparser
        description = '''The corpus subcommand has a number of subcommands of its own, including:
            list\t-\tlists all available corpora in dsrt's library
            add\t-\tadds a corpus to dsrt's library'''

        parser = argparse.ArgumentParser(description=description)
        self.init_corpus_args(parser)

        # parse the args we got
        args = parser.parse_args(sys.argv[2:3])

        corpus_command = 'corpus_' + args.corpus_command

        if not hasattr(self, corpus_command):
            print('Unrecognized corpus command.')
            parser.print_help()
            exit(1)

        getattr(self, corpus_command)()
1138710
def load(self, arguments):
        ""
        features = arguments[1:-1]
        list(map(self.load_feature, features))
538815
def read_corpus(file_name):
    
    with io.open(file_name, encoding='utf-8') as data_file:
        return yaml.load(data_file)
115847
def load_data_and_labels():
    
    # download dataset
    get_chinese_text()

    # Load data from files
    positive_examples = list(codecs.open("./data/pos.txt", "r", "utf-8").readlines())
    positive_examples = [s.strip() for s in positive_examples]
    positive_examples = [pe for pe in positive_examples if len(pe) < 100]
    negative_examples = list(codecs.open("./data/neg.txt", "r", "utf-8").readlines())
    negative_examples = [s.strip() for s in negative_examples]
    negative_examples = [ne for ne in negative_examples if len(ne) < 100]
    # Split by words
    x_text = positive_examples + negative_examples
    # x_text = [clean_str(sent) for sent in x_text]
    x_text = [list(s) for s in x_text]

    # Generate labels
    positive_labels = [[0, 1] for _ in positive_examples]
    negative_labels = [[1, 0] for _ in negative_examples]
    y = np.concatenate([positive_labels, negative_labels], 0)
    return [x_text, y]
34604
def generate_corpus(self, text):
        
        if isinstance(text, str):
            sentences = self.sentence_split(text)
        else:
            sentences = []
            for line in text:
                sentences += self.sentence_split(line)
        passing = filter(self.test_sentence_input, sentences)
        runs = map(self.word_split, passing)
        return runs
414989
def load(fname):
    """"""

    content = _open(fname).read()
    if PY2:
      state = pickle.loads(content)
    else:
      state = pickle.loads(content, encoding='latin1')
    voc, vec = state
    if len(voc) == 2:
      words, counts = voc
      word_count = dict(zip(words, counts))
      vocab = CountedVocabulary(word_count=word_count)
    else:
      vocab = OrderedVocabulary(voc)
    return Embedding(vocabulary=vocab, vectors=vec)
642380
def _load_vocab_file(vocab_file, reserved_tokens=None):
  """"""
  if reserved_tokens is None:
    reserved_tokens = RESERVED_TOKENS

  subtoken_list = []
  with tf.gfile.Open(vocab_file, mode="r") as f:
    for line in f:
      subtoken = _native_to_unicode(line.strip())
      subtoken = subtoken[1:-1]  # Remove surrounding single-quotes
      if subtoken in reserved_tokens:
        continue
      subtoken_list.append(_native_to_unicode(subtoken))
  return reserved_tokens + subtoken_list
1368993
def _assemble_corpus_string(self, corpus):
        """"""

        if corpus == 'phi5':
            filepaths = assemble_phi5_author_filepaths()
            file_cleaner = phi5_plaintext_cleanup
        elif corpus == 'tlg':
            filepaths = assemble_tlg_author_filepaths()
            file_cleaner = tlg_plaintext_cleanup

        for filepath in filepaths:
            with open(filepath) as file_open:
                file_read = file_open.read().lower()
            file_clean = file_cleaner(file_read)
            yield file_clean
1281987
def load_transliteration_table(lang="en", version="2"):
  
  src_dir = "transliteration{}".format(version)
  p = locate_resource(src_dir, lang)
  file_handler = _open(p)
  return pickle.load(file_handler)
399956
async def load(self,
                   file_path,
                   locale=None,
                   key: int = 0,
                   pos: int = 1,
                   neg: Optional[ColRanges] = None):
        

        if neg is None:
            neg: ColRanges = [(2, None)]

        await self.start(file_path, locale, kwargs={
            'key': key,
            'pos': pos,
            'neg': neg,
        })
657299
def loadtxt_asdict(fn, **kwargs):
    """"""

    kwargs.update(unpack=True)
    d = loadtxt(fn, **kwargs)
    if len(np.shape(d)) == 2:
        keys = kwargs.get('usecols', None) or range(len(d))
        D = dict([(k, v) for k, v in zip(keys, d)])
    elif len(np.shape(d)) == 1:
        keys = kwargs.get('usecols', None) or [0]
        D = dict([(keys[0], d)])
    else:
        raise Exception('Unknown dimension of loaded data.')

    return D
378136
def load_common(self, wlist):
        """"""
        self.com_dict = os.path.join(self.base_dir, 'dicts', 'EN_vocab.txt')
        with open(self.com_dict) as words_file:
            data = words_file.read()
        self.common_words = set(data.splitlines())
        if wlist:
            new_words = ''
            for wl in wlist:
                with open(wl) as f:
                    new_words += f.read()
            if new_words:
                new_dict = set(data.splitlines())
                self.common_words.update(new_dict)
394198
def _maybe_download_corpus(tmp_dir):
  
  corpus_url = ("http://www.statmt.org/lm-benchmark/"
                "1-billion-word-language-modeling-benchmark-r13output.tar.gz")
  corpus_filename = os.path.basename(corpus_url)
  corpus_filepath = os.path.join(tmp_dir, corpus_filename)
  if not os.path.exists(corpus_filepath):
    generator_utils.maybe_download(tmp_dir, corpus_filename, corpus_url)
    with tarfile.open(corpus_filepath, "r:gz") as corpus_tar:
      corpus_tar.extractall(tmp_dir)
519704
def generate_subtitles( # pylint: disable=too-many-locals,too-many-arguments
        source_path,
        output=None,
        concurrency=DEFAULT_CONCURRENCY,
        src_language=DEFAULT_SRC_LANGUAGE,
        dst_language=DEFAULT_DST_LANGUAGE,
        subtitle_file_format=DEFAULT_SUBTITLE_FORMAT,
        api_key=None,
    ):
    
    audio_filename, audio_rate = extract_audio(source_path)

    regions = find_speech_regions(audio_filename)

    pool = multiprocessing.Pool(concurrency)
    converter = FLACConverter(source_path=audio_filename)
    recognizer = SpeechRecognizer(language=src_language, rate=audio_rate,
                                  api_key=GOOGLE_SPEECH_API_KEY)

    transcripts = []
    if regions:
        try:
            widgets = ["Converting speech regions to FLAC files: ", Percentage(), ' ', Bar(), ' ',
                       ETA()]
            pbar = ProgressBar(widgets=widgets, maxval=len(regions)).start()
            extracted_regions = []
            for i, extracted_region in enumerate(pool.imap(converter, regions)):
                extracted_regions.append(extracted_region)
                pbar.update(i)
            pbar.finish()

            widgets = ["Performing speech recognition: ", Percentage(), ' ', Bar(), ' ', ETA()]
            pbar = ProgressBar(widgets=widgets, maxval=len(regions)).start()

            for i, transcript in enumerate(pool.imap(recognizer, extracted_regions)):
                transcripts.append(transcript)
                pbar.update(i)
            pbar.finish()

            if src_language.split("-")[0] != dst_language.split("-")[0]:
                if api_key:
                    google_translate_api_key = api_key
                    translator = Translator(dst_language, google_translate_api_key,
                                            dst=dst_language,
                                            src=src_language)
                    prompt = "Translating from {0} to {1}: ".format(src_language, dst_language)
                    widgets = [prompt, Percentage(), ' ', Bar(), ' ', ETA()]
                    pbar = ProgressBar(widgets=widgets, maxval=len(regions)).start()
                    translated_transcripts = []
                    for i, transcript in enumerate(pool.imap(translator, transcripts)):
                        translated_transcripts.append(transcript)
                        pbar.update(i)
                    pbar.finish()
                    transcripts = translated_transcripts
                else:
                    print(
                        "Error: Subtitle translation requires specified Google Translate API key. "
                        "See --help for further information."
                    )
                    return 1

        except KeyboardInterrupt:
            pbar.finish()
            pool.terminate()
            pool.join()
            print("Cancelling transcription")
            raise

    timed_subtitles = [(r, t) for r, t in zip(regions, transcripts) if t]
    formatter = FORMATTERS.get(subtitle_file_format)
    formatted_subtitles = formatter(timed_subtitles)

    dest = output

    if not dest:
        base = os.path.splitext(source_path)[0]
        dest = "{base}.{format}".format(base=base, format=subtitle_file_format)

    with open(dest, 'wb') as output_file:
        output_file.write(formatted_subtitles.encode("utf-8"))

    os.remove(audio_filename)

    return dest
731110
def loadLanguage(filename) :
	''''''

	L = { "consonants" : [], "vowels" : [], "onsets" : [] }
	
	f = open(filename, "r")
	section = None
	for line in f :
		line = line.strip()
		if line in ("[consonants]", "[vowels]", "[onsets]") :
			section = line[1:-1]
		elif section == None :
			raise ValueError, "File must start with a section header such as [consonants]."
		elif not section in L :
			raise ValueError, "Invalid section: " + section
		else :
			L[section].append(line)
			
	for section in "consonants", "vowels", "onsets" :
		if len(L[section]) == 0 :
			raise ValueError, "File does not contain any consonants, vowels, or onsets."
			
	return L