Create the dictionary of words, and grade dictionary, for the Dale-Chall readability test.
18254
def load_dale_chall(self):
        """"""
        self.dale_chall_dict = os.path.join(self.base_dir, 'dicts', 'dale_chall.txt')
        with open(self.dale_chall_dict) as words_file:
            data = words_file.read()
        self.dale_chall_words = set(data.splitlines())
384999
def setup_data(X, y, tokenizer, proc_data_path, **kwargs):
    
    # only build vocabulary once (e.g. training data)
    train = not tokenizer.has_vocab
    if train:
        tokenizer.build_vocab(X)

    process_save(X, y, tokenizer, proc_data_path,
                 train=train, **kwargs)
    return tokenizer
1083426
def _build_vocab(self, sentences, markers=[]):
        
        from snorkel.learning.pytorch.rnn.utils import SymbolTable

        vocab = Counter()
        for sent in sentences:
            for w in sent:
                vocab[w] += 1
        word_dict = SymbolTable()
        list(map(word_dict.get, vocab))
        list(map(word_dict.get, markers))
        return word_dict
117868
def _build_dictionary(self, results):
        
        foreign = self._foreign_key

        dictionary = {}

        for result in results:
            key = getattr(result.pivot, foreign)
            if key not in dictionary:
                dictionary[key] = []

            dictionary[key].append(result)

        return dictionary
965248
def _init_vocab(self, token_generator, add_reserved_tokens=True):
    """"""

    self._id_to_token = {}
    non_reserved_start_index = 0

    if add_reserved_tokens:
      self._id_to_token.update(enumerate(RESERVED_TOKENS))
      non_reserved_start_index = len(RESERVED_TOKENS)

    self._id_to_token.update(
        enumerate(token_generator, start=non_reserved_start_index))

    # _token_to_id is the reverse of _id_to_token
    self._token_to_id = dict((v, k)
                             for k, v in six.iteritems(self._id_to_token))
1369594
def to_dict(self):
        
        d = {}
        for word, idx in self.vocab.iteritems():
            d[word] = self.array[idx].tolist()
        return d
514347
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'text') and self.text is not None:
            _dict['text'] = self.text
        if hasattr(self, 'score') and self.score is not None:
            _dict['score'] = self.score
        return _dict
540147
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'trait_id' in _dict:
            args['trait_id'] = _dict.get('trait_id')
        else:
            raise ValueError(
                'Required property \'trait_id\' not present in Trait JSON')
        if 'name' in _dict:
            args['name'] = _dict.get('name')
        else:
            raise ValueError(
                'Required property \'name\' not present in Trait JSON')
        if 'category' in _dict:
            args['category'] = _dict.get('category')
        else:
            raise ValueError(
                'Required property \'category\' not present in Trait JSON')
        if 'percentile' in _dict:
            args['percentile'] = _dict.get('percentile')
        else:
            raise ValueError(
                'Required property \'percentile\' not present in Trait JSON')
        if 'raw_score' in _dict:
            args['raw_score'] = _dict.get('raw_score')
        if 'significant' in _dict:
            args['significant'] = _dict.get('significant')
        if 'children' in _dict:
            args['children'] = [
                Trait._from_dict(x) for x in (_dict.get('children'))
            ]
        return cls(**args)
310772
def makedict(self, dictfile, fnamefobject):
        """"""
        #fname = './exapmlefiles/5ZoneDD.idf'
        #fname = './1ZoneUncontrolled.idf'
        if isinstance(dictfile, Idd):
            localidd = copy.deepcopy(dictfile)
            dt, dtls = localidd.dt, localidd.dtls
        else:
            dt, dtls = self.initdict(dictfile)
        # astr = mylib2.readfile(fname)
        astr = fnamefobject.read()
        try:
            astr = astr.decode('ISO-8859-2')
        except AttributeError:
            pass
        fnamefobject.close()
        nocom = removecomment(astr, '!')
        idfst = nocom
        # alist = string.split(idfst, ';')
        alist = idfst.split(';')
        lss = []
        for element in alist:
            # lst = string.split(element, ',')
            lst = element.split(',')
            lss.append(lst)

        for i in range(0, len(lss)):
            for j in range(0, len(lss[i])):
                lss[i][j] = lss[i][j].strip()

        for element in lss:
            node = element[0].upper()
            if node in dt:
                # stuff data in this key
                dt[node.upper()].append(element)
            else:
                # scream
                if node == '':
                    continue
                print('this node -%s-is not present in base dictionary' %
                      (node))

        self.dt, self.dtls = dt, dtls
        return dt, dtls
42154
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'final' in _dict or 'final_results' in _dict:
            args['final_results'] = _dict.get('final') or _dict.get(
                'final_results')
        else:
            raise ValueError(
                'Required property \'final\' not present in SpeechRecognitionResult JSON'
            )
        if 'alternatives' in _dict:
            args['alternatives'] = [
                SpeechRecognitionAlternative._from_dict(x)
                for x in (_dict.get('alternatives'))
            ]
        else:
            raise ValueError(
                'Required property \'alternatives\' not present in SpeechRecognitionResult JSON'
            )
        if 'keywords_result' in _dict:
            args['keywords_result'] = _dict.get('keywords_result')
        if 'word_alternatives' in _dict:
            args['word_alternatives'] = [
                WordAlternativeResults._from_dict(x)
                for x in (_dict.get('word_alternatives'))
            ]
        return cls(**args)
786428
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'processed_language' in _dict:
            args['processed_language'] = _dict.get('processed_language')
        else:
            raise ValueError(
                'Required property \'processed_language\' not present in Profile JSON'
            )
        if 'word_count' in _dict:
            args['word_count'] = _dict.get('word_count')
        else:
            raise ValueError(
                'Required property \'word_count\' not present in Profile JSON')
        if 'word_count_message' in _dict:
            args['word_count_message'] = _dict.get('word_count_message')
        if 'personality' in _dict:
            args['personality'] = [
                Trait._from_dict(x) for x in (_dict.get('personality'))
            ]
        else:
            raise ValueError(
                'Required property \'personality\' not present in Profile JSON')
        if 'needs' in _dict:
            args['needs'] = [Trait._from_dict(x) for x in (_dict.get('needs'))]
        else:
            raise ValueError(
                'Required property \'needs\' not present in Profile JSON')
        if 'values' in _dict:
            args['values'] = [
                Trait._from_dict(x) for x in (_dict.get('values'))
            ]
        else:
            raise ValueError(
                'Required property \'values\' not present in Profile JSON')
        if 'behavior' in _dict:
            args['behavior'] = [
                Behavior._from_dict(x) for x in (_dict.get('behavior'))
            ]
        if 'consumption_preferences' in _dict:
            args['consumption_preferences'] = [
                ConsumptionPreferencesCategory._from_dict(x)
                for x in (_dict.get('consumption_preferences'))
            ]
        if 'warnings' in _dict:
            args['warnings'] = [
                Warning._from_dict(x) for x in (_dict.get('warnings'))
            ]
        else:
            raise ValueError(
                'Required property \'warnings\' not present in Profile JSON')
        return cls(**args)
432416
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'word') and self.word is not None:
            _dict['word'] = self.word
        if hasattr(self, 'sounds_like') and self.sounds_like is not None:
            _dict['sounds_like'] = self.sounds_like
        if hasattr(self, 'display_as') and self.display_as is not None:
            _dict['display_as'] = self.display_as
        return _dict
403406
def _get_or_generate_vocab(tmp_dir, vocab_filename, vocab_size):
  """"""
  vocab_filepath = os.path.join(tmp_dir, vocab_filename)
  print('Vocab file written to: ' + vocab_filepath)

  if tf.gfile.Exists(vocab_filepath):
    gs = text_encoder.SubwordTextEncoder(vocab_filepath)
    return gs
  example_file = os.path.join(tmp_dir, _EXAMPLES_FILE)
  gs = text_encoder.SubwordTextEncoder()
  token_counts = tokenizer.corpus_token_counts(
      example_file, corpus_max_lines=1000000)
  gs = gs.build_to_target_size(
      vocab_size, token_counts, min_val=1, max_val=1e3)
  gs.store_to_file(vocab_filepath)
  return gs
131207
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'word' in _dict:
            args['word'] = _dict.get('word')
        else:
            raise ValueError(
                'Required property \'word\' not present in Word JSON')
        if 'sounds_like' in _dict:
            args['sounds_like'] = _dict.get('sounds_like')
        else:
            raise ValueError(
                'Required property \'sounds_like\' not present in Word JSON')
        if 'display_as' in _dict:
            args['display_as'] = _dict.get('display_as')
        else:
            raise ValueError(
                'Required property \'display_as\' not present in Word JSON')
        if 'count' in _dict:
            args['count'] = _dict.get('count')
        else:
            raise ValueError(
                'Required property \'count\' not present in Word JSON')
        if 'source' in _dict:
            args['source'] = _dict.get('source')
        else:
            raise ValueError(
                'Required property \'source\' not present in Word JSON')
        if 'error' in _dict:
            args['error'] = [
                WordError._from_dict(x) for x in (_dict.get('error'))
            ]
        return cls(**args)
946130
def _prepare_lambada_data(tmp_dir, data_dir, vocab_size, vocab_filename):
  

  if not tf.gfile.Exists(data_dir):
    tf.gfile.MakeDirs(data_dir)

  file_path = generator_utils.maybe_download(tmp_dir, _TAR, _URL)
  tar_all = tarfile.open(file_path)
  tar_all.extractall(tmp_dir)
  tar_all.close()
  tar_train = tarfile.open(os.path.join(tmp_dir, "train-novels.tar"))
  tar_train.extractall(tmp_dir)
  tar_train.close()

  vocab_path = os.path.join(data_dir, vocab_filename)
  if not tf.gfile.Exists(vocab_path):
    with tf.gfile.GFile(os.path.join(tmp_dir, _VOCAB), "r") as infile:
      reader = csv.reader(infile, delimiter="\t")
      words = [row[0] for row in reader]
      words = [_UNK] + words[:vocab_size]
    with tf.gfile.GFile(vocab_path, "w") as outfile:
      outfile.write("\n".join(words))
1182731
def cli():
    """"""
    args = docopt.docopt(__doc__, version=__VERSION__)
    secure = args['--secure']
    numberofwords = int(args['<numberofwords>'])

    dictpath = args['--dict']
    if dictpath is not None:
        dictfile = open(dictpath)
    else:
        dictfile = load_stream('words.txt')
    with dictfile:
        wordlist = read_wordlist(dictfile)

    words = generate_words(numberofwords, wordlist, secure=secure)
    print(' '.join(words))
1317028
def _to_dict(self):
        """"""
        _dict = {}
        if hasattr(self, 'from_') and self.from_ is not None:
            _dict['from'] = self.from_
        if hasattr(self, 'to') and self.to is not None:
            _dict['to'] = self.to
        if hasattr(self, 'speaker') and self.speaker is not None:
            _dict['speaker'] = self.speaker
        if hasattr(self, 'confidence') and self.confidence is not None:
            _dict['confidence'] = self.confidence
        if hasattr(self, 'final_results') and self.final_results is not None:
            _dict['final'] = self.final_results
        return _dict