Construct dictionary DAWG from tokenized words.
348891
def build(self, words):
        """"""
        words = [self._normalize(tokens) for tokens in words]
        self._dawg = dawg.CompletionDAWG(words)
        self._loaded_model = True
58401
def _build_vocab(filename, vocab_dir, vocab_name):
  
  vocab_path = os.path.join(vocab_dir, vocab_name)
  if not tf.gfile.Exists(vocab_path):
    with tf.gfile.GFile(filename, "r") as f:
      data = f.read().split()
    counter = collections.Counter(data)
    count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
    words, _ = list(zip(*count_pairs))
    encoder = text_encoder.TokenTextEncoder(None, vocab_list=words)
    encoder.store_to_file(vocab_path)
  else:
    encoder = text_encoder.TokenTextEncoder(vocab_path)
  return encoder
1035860
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'query_id' in _dict:
            args['query_id'] = _dict.get('query_id')
        if 'natural_language_query' in _dict:
            args['natural_language_query'] = _dict.get('natural_language_query')
        if 'filter' in _dict:
            args['filter'] = _dict.get('filter')
        if 'examples' in _dict:
            args['examples'] = [
                TrainingExample._from_dict(x) for x in (_dict.get('examples'))
            ]
        return cls(**args)
1139761
def dictionize(fields: Sequence, records: Sequence) -> Generator:
    """"""

    return (dict(zip(fields, rec)) for rec in records)
1119872
def _token_to_subwords(self, token):
    """"""
    subwords = []

    start = 0
    while start < len(token):
      subword = None
      for end in range(
          min(len(token), start + self._max_subword_len), start, -1):
        candidate = token[start:end]
        if (candidate in self._subword_to_id or
            candidate == _UNDERSCORE_REPLACEMENT):
          subword = candidate
          subwords.append(subword)
          start = end
          break
      # No subword match found. Consume a single (unicode) character.
      if subword is None:
        subwords.append(token[start])
        start += 1

    return subwords
36846
def phrases_reduce(key, values):
  """"""
  if len(values) < 10:
    return
  counts = {}
  for filename in values:
    counts[filename] = counts.get(filename, 0) + 1

  words = re.sub(r":", " ", key)
  threshold = len(values) / 2
  for filename, count in counts.items():
    if count > threshold:
      yield "%s:%s\n" % (words, filename)
46833
def stem(self, text):
        """"""
        normalizedText = TextNormalizer.normalize_text(text)

        words = normalizedText.split(' ')
        stems = []

        for word in words:
            stems.append(self.stem_word(word))

        return ' '.join(stems)
1005931
def setup_dictionary(self, task):
        """"""

        dictionary_options = task.get('dictionary', {})
        output = os.path.abspath(dictionary_options.get('output', self.dict_bin))
        lang = dictionary_options.get('lang', 'en_US')
        wordlists = dictionary_options.get('wordlists', [])
        if lang and wordlists:
            self.compile_dictionary(lang, dictionary_options.get('wordlists', []), None, output)
        else:
            output = None
        return output
315010
def from_files(cls, directory: str) -> 'Vocabulary':
        
        logger.info("Loading token dictionary from %s.", directory)
        with codecs.open(os.path.join(directory, NAMESPACE_PADDING_FILE), 'r', 'utf-8') as namespace_file:
            non_padded_namespaces = [namespace_str.strip() for namespace_str in namespace_file]

        vocab = cls(non_padded_namespaces=non_padded_namespaces)

        # Check every file in the directory.
        for namespace_filename in os.listdir(directory):
            if namespace_filename == NAMESPACE_PADDING_FILE:
                continue
            if namespace_filename.startswith("."):
                continue
            namespace = namespace_filename.replace('.txt', '')
            if any(namespace_match(pattern, namespace) for pattern in non_padded_namespaces):
                is_padded = False
            else:
                is_padded = True
            filename = os.path.join(directory, namespace_filename)
            vocab.set_from_file(filename, is_padded, namespace=namespace)

        return vocab
150958
def tokenize(self, path):
        """"""
        assert os.path.exists(path)
        # Add words to the dictionary
        with open(path, 'r') as f:
            tokens = 0
            for line in f:
                words = line.split() + ['<eos>']
                tokens += len(words)
                for word in words:
                    self.dictionary.add_word(word)

        # Tokenize file content
        with open(path, 'r') as f:
            ids = np.zeros((tokens,), dtype='int32')
            token = 0
            for line in f:
                words = line.split() + ['<eos>']
                for word in words:
                    ids[token] = self.dictionary.word2idx[word]
                    token += 1

        return mx.nd.array(ids, dtype='int32')
651628
def from_dict(raw_data):
        """"""
        try:
            definition = raw_data['definition']
            term_id = raw_data['id']
            image = Image.from_dict(raw_data['image'])
            rank = raw_data['rank']
            term = raw_data['term']
            return Term(definition, term_id, image, rank, term)
        except KeyError:
            raise ValueError('Unexpected term json structure')
513026
def cfg2dict(cfg):
    
    d = {}
    for section in cfg.sections():
        d[section] = dict(cfg.items(section))
    return d
98606
def read_dictionary_file(dictionary_path):
    """"""
    try:
        return _user_dictionary_cache[dictionary_path]
    except KeyError:
        if dictionary_path and os.path.exists(dictionary_path):
            with open(dictionary_path, "rt") as dict_f:
                words = set(re.findall(r"(\w[\w']*\w|\w)",
                                       " ".join(dict_f.read().splitlines())))
                return words

        return set()
700446
def snli_token_generator(tmp_dir, train, vocab_size):
  """"""
  _download_and_parse_dataset(tmp_dir, train)

  symbolizer_vocab = _get_or_generate_vocab(
      tmp_dir, 'vocab.subword_text_encoder', vocab_size)

  file_name = 'train' if train else 'dev'
  data_file = os.path.join(tmp_dir, file_name + '.txt')
  with tf.gfile.GFile(data_file, mode='r') as f:
    for line in f:
      sent1, sent2, label = line.strip().split('\t')
      sent1_enc = symbolizer_vocab.encode(sent1)
      sent2_enc = symbolizer_vocab.encode(sent2)

      inputs = sent1_enc + [_SEP] + sent2_enc + [_EOS]
      yield {
          'inputs': inputs,
          'targets': [_LABEL_TO_ID[label]],
      }
1157804
def create(technical_terms_filename, spellchecker_cache_path):
    """"""
    user_dictionary = os.path.join(os.getcwd(), "DICTIONARY")
    user_words = read_dictionary_file(user_dictionary)

    technical_terms_set = set(user_words)

    if technical_terms_filename:
        with open(technical_terms_filename) as tech_tf:
            technical_terms_set |= set(tech_tf.read().splitlines())

    return Dictionary(technical_terms_set,
                      "technical_words",
                      dictionary_sources=[technical_terms_filename,
                                          user_dictionary],
                      cache=spellchecker_cache_path)
591247
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'id' in _dict:
            args['id'] = _dict.get('id')
        else:
            raise ValueError(
                'Required property \'id\' not present in RecognitionJob JSON')
        if 'status' in _dict:
            args['status'] = _dict.get('status')
        else:
            raise ValueError(
                'Required property \'status\' not present in RecognitionJob JSON'
            )
        if 'created' in _dict:
            args['created'] = _dict.get('created')
        else:
            raise ValueError(
                'Required property \'created\' not present in RecognitionJob JSON'
            )
        if 'updated' in _dict:
            args['updated'] = _dict.get('updated')
        if 'url' in _dict:
            args['url'] = _dict.get('url')
        if 'user_token' in _dict:
            args['user_token'] = _dict.get('user_token')
        if 'results' in _dict:
            args['results'] = [
                SpeechRecognitionResults._from_dict(x)
                for x in (_dict.get('results'))
            ]
        if 'warnings' in _dict:
            args['warnings'] = _dict.get('warnings')
        return cls(**args)
447011
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'examples' in _dict:
            args['examples'] = [
                TrainingExample._from_dict(x) for x in (_dict.get('examples'))
            ]
        return cls(**args)