Load from a vocab file.
777371
def _load_from_file(self, filename):
    """"""
    if not tf.gfile.Exists(filename):
      raise ValueError("File %s not found" % filename)
    with tf.gfile.Open(filename) as f:
      self._load_from_file_object(f)
60668
public void buildVocab() {


        val constructor = new VocabConstructor.Builder<T>().addSource(iterator, minWordFrequency)
                        .setTargetVocabCache(vocab).fetchLabels(trainSequenceVectors).setStopWords(stopWords)
                        .enableScavenger(enableScavenger).setEntriesLimit(vocabLimit)
                        .allowParallelTokenization(configuration.isAllowParallelTokenization())
                        .setUnk(useUnknown && unknownElement != null ? unknownElement : null).build();

        if (existingModel != null && lookupTable instanceof InMemoryLookupTable
                        && existingModel.lookupTable() instanceof InMemoryLookupTable) {
            log.info("Merging existing vocabulary into the current one...");
            /*
                if we have existing model defined, we're forced to fetch labels only.
                the rest of vocabulary & weights should be transferred from existing model
             */

            constructor.buildMergedVocabulary(existingModel, true);

            /*
                Now we have vocab transferred, and we should transfer syn0 values into lookup table
             */
            ((InMemoryLookupTable<VocabWord>) lookupTable)
                            .consume((InMemoryLookupTable<VocabWord>) existingModel.lookupTable());
        } else {
            log.info("Starting vocabulary building...");
            // if we don't have existing model defined, we just build vocabulary


            constructor.buildJointVocabulary(false, true);

            /*
            if (useUnknown && unknownElement != null && !vocab.containsWord(unknownElement.getLabel())) {
                log.info("Adding UNK element...");
                unknownElement.setSpecial(true);
                unknownElement.markAsLabel(false);
                unknownElement.setIndex(vocab.numWords());
                vocab.addToken(unknownElement);
            }
            */


            // check for malformed inputs. if numWords/numSentences ratio is huge, then user is passing something weird
            if (vocab.numWords() / constructor.getNumberOfSequences() > 1000) {
                log.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
                log.warn("!                                                                                       !");
                log.warn("! Your input looks malformed: number of sentences is too low, model accuracy may suffer !");
                log.warn("!                                                                                       !");
                log.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
            }
        }


    }
592119
def get_vocab(self, vocab_name, **kwargs):
        
        vocab_dict = self.__get_vocab_dict__(vocab_name, **kwargs)

        filepaths = list(set([os.path.join(self.cache_dir,
                                           vocab_dict['filename']),
                              os.path.join(self.vocab_dir,
                                           vocab_dict['filename'])]))
        for path in filepaths:
            if os.path.exists(path):
                with open(path, 'rb') as f_obj:
                    vocab_dict.update({"name": vocab_name,
                                       "data": f_obj.read(),
                                       "modified": os.path.getmtime(path)})
                return vocab_dict
        download_locs = make_list(vocab_dict.get('download',[]))
        for loc in download_locs:

            loc_web = urllib.request.urlopen(loc)
            # loc_file_date = date_parse(loc_web.info()['Last-Modified'])
            urllib.request.urlretrieve(loc, filepaths[0])
            with open(filepaths[0], 'rb') as f_obj:
                vocab_dict.update({"name": vocab_name,
                                   "data": f_obj.read(),
                                   "modified": os.path.getmtime(filepaths[0])})
                return vocab_dict
1226005
def _load_entries(self):
        """"""

        rel_path = os.path.join('~','cltk_data',
                                'french',
                                'text','french_data_cltk'
                                ,'entries.py')
        path = os.path.expanduser(rel_path)
        #logger.info('Loading entries. This may take a minute.')
        loader = importlib.machinery.SourceFileLoader('entries', path)
        module = loader.load_module()
        entries = module.entries
        return entries
371135
private function loadFullDictionary(): void
    {
        $file = __DIR__ . '/../Dictionary/' . $this->language;
        if (!file_exists($file)) {
            $file = __DIR__ . '/../Dictionary/en_US';
        }

        self::$dictionary = array_filter(explode(PHP_EOL, file_get_contents($file)));
    }
587146
def get_vocabulary(preprocess_output_dir, name):
  
  vocab_file = os.path.join(preprocess_output_dir, CATEGORICAL_ANALYSIS % name)
  if not file_io.file_exists(vocab_file):
    raise ValueError('File %s not found in %s' %
                     (CATEGORICAL_ANALYSIS % name, preprocess_output_dir))

  labels = python_portable_string(
      file_io.read_file_to_string(vocab_file)).split('\n')
  label_values = [x for x in labels if x]  # remove empty lines

  return label_values
871555
def convert_vocab(vocab_file):
    """"""
    original_vocab = load_vocab(vocab_file)
    token_to_idx = dict(original_vocab)
    num_tokens = len(token_to_idx)
    idx_to_token = [None] * len(original_vocab)
    for word in original_vocab:
        idx = int(original_vocab[word])
        idx_to_token[idx] = word

    def swap(token, target_idx, token_to_idx, idx_to_token, swap_idx):
        original_idx = token_to_idx[token]
        original_token = idx_to_token[target_idx]
        token_to_idx[token] = target_idx
        token_to_idx[original_token] = original_idx
        idx_to_token[target_idx] = token
        idx_to_token[original_idx] = original_token
        swap_idx.append((original_idx, target_idx))

    reserved_tokens = [gluonnlp.vocab.BERTVocab.PADDING_TOKEN, gluonnlp.vocab.BERTVocab.CLS_TOKEN,
                       gluonnlp.vocab.BERTVocab.SEP_TOKEN, gluonnlp.vocab.BERTVocab.MASK_TOKEN]

    unknown_token = gluonnlp.vocab.BERTVocab.UNKNOWN_TOKEN
    padding_token = gluonnlp.vocab.BERTVocab.PADDING_TOKEN
    swap_idx = []
    assert unknown_token in token_to_idx
    assert padding_token in token_to_idx
    swap(unknown_token, 0, token_to_idx, idx_to_token, swap_idx)
    for i, token in enumerate(reserved_tokens):
        swap(token, i + 1, token_to_idx, idx_to_token, swap_idx)

    # sanity checks
    assert len(token_to_idx) == num_tokens
    assert len(idx_to_token) == num_tokens
    assert None not in idx_to_token
    assert len(set(idx_to_token)) == num_tokens

    bert_vocab_dict = {}
    bert_vocab_dict['idx_to_token'] = idx_to_token
    bert_vocab_dict['token_to_idx'] = token_to_idx
    bert_vocab_dict['reserved_tokens'] = reserved_tokens
    bert_vocab_dict['unknown_token'] = unknown_token
    bert_vocab_dict['padding_token'] = padding_token
    bert_vocab_dict['bos_token'] = None
    bert_vocab_dict['eos_token'] = None
    bert_vocab_dict['mask_token'] = gluonnlp.vocab.BERTVocab.MASK_TOKEN
    bert_vocab_dict['sep_token'] = gluonnlp.vocab.BERTVocab.SEP_TOKEN
    bert_vocab_dict['cls_token'] = gluonnlp.vocab.BERTVocab.CLS_TOKEN
    json_str = json.dumps(bert_vocab_dict)
    converted_vocab = gluonnlp.vocab.BERTVocab.from_json(json_str)
    return converted_vocab, swap_idx
633603
def from_mmap(cls, fname):
        
        memmaped = joblib.load(fname, mmap_mode="r+")
        return cls(vocab=memmaped.vocab, vectors=memmaped.vectors)
1091530
def loadDict(filename):
    """"""
    filename = os.path.expanduser(filename)
    if not splitext(filename)[1]: filename += ".bpickle"
    f = None
    try:
        f = open(filename, "rb")
        varH = cPickle.load(f)
    finally:
        if f: f.close()
    return varH
112779
def load_nouns(self, file):
        
        with open(os.path.join(main_dir, file + '.dat'), 'r') as f:
            self.nouns = json.load(f)
285860
def get_docs(self, vocab):
        """"""
        for string in self.strings:
            vocab[string]
        orth_col = self.attrs.index(ORTH)
        for tokens, spaces in zip(self.tokens, self.spaces):
            words = [vocab.strings[orth] for orth in tokens[:, orth_col]]
            doc = Doc(vocab, words=words, spaces=spaces)
            doc = doc.from_array(self.attrs, tokens)
            yield doc
913327
def collect_vocab(qp_pairs):
    '''
    
    '''
    vocab = set()
    for qp_pair in qp_pairs:
        for word in qp_pair['question_tokens']:
            vocab.add(word['word'])
        for word in qp_pair['passage_tokens']:
            vocab.add(word['word'])
    return vocab
1289422
def read_vocab_file(file_path):
  
  with file_io.FileIO(file_path, 'r') as f:
    vocab_pd = pd.read_csv(
        f,
        header=None,
        names=['vocab', 'count'],
        dtype=str,  # Prevent pd from converting numerical categories.
        na_filter=False)  # Prevent pd from converting 'NA' to a NaN.

  vocab = vocab_pd['vocab'].tolist()
  ex_count = vocab_pd['count'].astype(int).tolist()

  return vocab, ex_count
1083426
def _build_vocab(self, sentences, markers=[]):
        
        from snorkel.learning.pytorch.rnn.utils import SymbolTable

        vocab = Counter()
        for sent in sentences:
            for w in sent:
                vocab[w] += 1
        word_dict = SymbolTable()
        list(map(word_dict.get, vocab))
        list(map(word_dict.get, markers))
        return word_dict
535405
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'words' in _dict:
            args['words'] = [Word._from_dict(x) for x in (_dict.get('words'))]
        else:
            raise ValueError(
                'Required property \'words\' not present in Words JSON')
        return cls(**args)
239175
def generate_lines_for_vocab(tmp_dir, sources, file_byte_budget=1e6):
  """"""
  tf.logging.info("Generating vocab from: %s", str(sources))
  for source in sources:
    url = source[0]
    filename = os.path.basename(url)
    compressed_file = maybe_download(tmp_dir, filename, url)

    for lang_file in source[1]:
      tf.logging.info("Reading file: %s" % lang_file)
      filepath = os.path.join(tmp_dir, lang_file)

      # Extract from tar if needed.
      if not tf.gfile.Exists(filepath):
        read_type = "r:gz" if filename.endswith("tgz") else "r"
        with tarfile.open(compressed_file, read_type) as corpus_tar:
          corpus_tar.extractall(tmp_dir)

      # For some datasets a second extraction is necessary.
      if lang_file.endswith(".gz"):
        new_filepath = os.path.join(tmp_dir, lang_file[:-3])
        if tf.gfile.Exists(new_filepath):
          tf.logging.info(
              "Subdirectory %s already exists, skipping unpacking" % filepath)
        else:
          tf.logging.info("Unpacking subdirectory %s" % filepath)
          gunzip_file(filepath, new_filepath)
        filepath = new_filepath

      with tf.gfile.GFile(filepath, mode="r") as source_file:
        file_byte_budget_ = file_byte_budget
        counter = 0
        countermax = int(source_file.size() / file_byte_budget_ / 2)
        for line in source_file:
          if counter < countermax:
            counter += 1
          else:
            if file_byte_budget_ <= 0:
              break
            line = line.strip()
            file_byte_budget_ -= len(line)
            counter = 0
            yield line
1054398
def load_word_file(filename):
    """"""
    words_file = resource_filename(__name__, "words/%s" % filename)
    handle = open(words_file, 'r')
    words = handle.readlines()
    handle.close()
    return words