nodes: 0, module; 1, function_definition; 2, function_name:compile_vocab; 3, parameters; 4, block; 5, identifier:docs; 6, default_parameter; 7, default_parameter; 8, default_parameter; 9, expression_statement; 10, expression_statement; 11, expression_statement; 12, try_statement; 13, for_statement; 14, return_statement; 15, identifier:limit; 16, float:1e6; 17, identifier:verbose; 18, integer:0; 19, identifier:tokenizer; 20, call; 21, comment:"""Get the set of words used anywhere in a sequence of documents and assign an integer id
  
      This vectorizer is much faster than the scikit-learn version (and only requires low/constant RAM ?).
  
      >>> gen = ('label: ' + chr(ord('A') + i % 3)*3 for i in range(11))
      >>> d = compile_vocab(gen, verbose=0)
      >>> d
      <gensim.corpora.dictionary.Dictionary ...>
      >>> print(d)
      Dictionary(4 unique tokens: [u'AAA', u'BBB', u'CCC', u'label'])
      >>> sorted(d.token2id.values())
      [0, 1, 2, 3]
      >>> sorted(d.token2id.keys())
      [u'AAA', u'BBB', u'CCC', u'label']
      """; 22, assignment; 23, assignment; 24, block; 25, except_clause; 26, pattern_list; 27, call; 28, comment:# if isinstance(doc, (tuple, list)) and len(doc) == 2 and isinstance(doc[1], int):; 29, comment:#     doc, score = docs; 30, block; 31, identifier:d; 32, identifier:Tokenizer; 33, argument_list; 34, identifier:tokenizer; 35, call; 36, identifier:d; 37, call; 38, expression_statement; 39, expression_statement; 40, tuple; 41, block; 42, identifier:i; 43, identifier:doc; 44, identifier:enumerate; 45, argument_list; 46, try_statement; 47, if_statement; 48, expression_statement; 49, if_statement; 50, keyword_argument; 51, keyword_argument; 52, keyword_argument; 53, identifier:make_tokenizer; 54, argument_list; 55, identifier:Dictionary; 56, argument_list; 57, assignment; 58, assignment; 59, identifier:AttributeError; 60, identifier:TypeError; 61, pass_statement; 62, identifier:docs; 63, comment:# in case docs is a values() queryset (dicts of records in a DB table); 64, block; 65, except_clause; 66, comparison_operator:i >= limit; 67, block; 68, call; 69, boolean_operator; 70, block; 71, identifier:stem; 72, None; 73, identifier:lower; 74, None; 75, identifier:strip; 76, None; 77, identifier:tokenizer; 78, identifier:limit; 79, call; 80, identifier:docs; 81, call; 82, expression_statement; 83, identifier:AttributeError; 84, comment:# doc already is a values_list; 85, block; 86, identifier:i; 87, identifier:limit; 88, break_statement; 89, attribute; 90, argument_list; 91, identifier:verbose; 92, not_operator; 93, expression_statement; 94, identifier:min; 95, argument_list; 96, attribute; 97, argument_list; 98, assignment; 99, if_statement; 100, identifier:d; 101, identifier:add_documents; 102, list; 103, binary_operator:i % 100; 104, call; 105, identifier:limit; 106, call; 107, identifier:docs; 108, identifier:iterator; 109, identifier:doc; 110, call; 111, not_operator; 112, block; 113, else_clause; 114, call; 115, identifier:i; 116, integer:100; 117, attribute; 118, argument_list; 119, attribute; 120, argument_list; 121, attribute; 122, argument_list; 123, call; 124, expression_statement; 125, block; 126, identifier:list; 127, argument_list; 128, identifier:log; 129, identifier:info; 130, call; 131, identifier:docs; 132, identifier:count; 133, identifier:doc; 134, identifier:values; 135, identifier:isinstance; 136, argument_list; 137, assignment; 138, expression_statement; 139, call; 140, attribute; 141, argument_list; 142, identifier:doc; 143, identifier:str; 144, identifier:doc; 145, call; 146, assignment; 147, identifier:tokenizer; 148, argument_list; 149, string; 150, identifier:format; 151, identifier:i; 152, subscript; 153, attribute; 154, argument_list; 155, identifier:doc; 156, call; 157, identifier:doc; 158, string_content:{}: {}; 159, call; 160, slice; 161, string; 162, identifier:join; 163, list_comprehension; 164, identifier:str; 165, argument_list; 166, identifier:repr; 167, argument_list; 168, integer:120; 169, string_content:; 170, call; 171, for_in_clause; 172, identifier:doc; 173, identifier:d; 174, identifier:str; 175, argument_list; 176, identifier:v; 177, identifier:doc; 178, identifier:v
code: def compile_vocab(docs, limit=1e6, verbose=0, tokenizer=Tokenizer(stem=None, lower=None, strip=None)):
      """Get the set of words used anywhere in a sequence of documents and assign an integer id
  
      This vectorizer is much faster than the scikit-learn version (and only requires low/constant RAM ?).
  
      >>> gen = ('label: ' + chr(ord('A') + i % 3)*3 for i in range(11))
      >>> d = compile_vocab(gen, verbose=0)
      >>> d
      <gensim.corpora.dictionary.Dictionary ...>
      >>> print(d)
      Dictionary(4 unique tokens: [u'AAA', u'BBB', u'CCC', u'label'])
      >>> sorted(d.token2id.values())
      [0, 1, 2, 3]
      >>> sorted(d.token2id.keys())
      [u'AAA', u'BBB', u'CCC', u'label']
      """
      tokenizer = make_tokenizer(tokenizer)
      d = Dictionary()
  
      try:
          limit = min(limit, docs.count())
          docs = docs.iterator()
      except (AttributeError, TypeError):
          pass
      for i, doc in enumerate(docs):
          # if isinstance(doc, (tuple, list)) and len(doc) == 2 and isinstance(doc[1], int):
          #     doc, score = docs
          try:
              # in case docs is a values() queryset (dicts of records in a DB table)
              doc = doc.values()
          except AttributeError:  # doc already is a values_list
              if not isinstance(doc, str):
                  doc = ' '.join([str(v) for v in doc])
              else:
                  doc = str(doc)
          if i >= limit:
              break
          d.add_documents([list(tokenizer(doc))])
          if verbose and not i % 100:
              log.info('{}: {}'.format(i, repr(d)[:120]))
      return d