nodes: 0, module; 1, function_definition; 2, function_name:_generate_subtokens; 3, parameters; 4, block; 5, identifier:token_counts; 6, identifier:alphabet; 7, identifier:min_count; 8, default_parameter; 9, default_parameter; 10, expression_statement; 11, if_statement; 12, comment:# Use alphabet set to create initial list of subtokens; 13, expression_statement; 14, expression_statement; 15, comment:# On each iteration, segment all words using the subtokens defined in; 16, comment:# subtoken_dict, count how often the resulting subtokens appear, and update; 17, comment:# the dictionary with subtokens w/ high enough counts.; 18, for_statement; 19, return_statement; 20, identifier:num_iterations; 21, integer:4; 22, identifier:reserved_tokens; 23, None; 24, comment:"""Create a list of subtokens in decreasing order of frequency.
  
    Args:
      token_counts: dict mapping str tokens -> int count
      alphabet: set of characters
      min_count: int minimum number of times a subtoken must appear before it is
        added to the vocabulary.
      num_iterations: int number of iterations to generate new tokens.
      reserved_tokens: list of tokens that will be added to the beginning to the
        returned subtoken list.
  
    Returns:
      Sorted list of subtokens (most frequent first)
    """; 25, comparison_operator:reserved_tokens is None; 26, block; 27, assignment; 28, assignment; 29, identifier:i; 30, call; 31, block; 32, identifier:subtoken_list; 33, identifier:reserved_tokens; 34, None; 35, expression_statement; 36, identifier:subtoken_list; 37, binary_operator:reserved_tokens + list(alphabet); 38, identifier:max_subtoken_length; 39, integer:1; 40, identifier:xrange; 41, argument_list; 42, expression_statement; 43, comment:# Generate new subtoken->id dictionary using the new subtoken list.; 44, expression_statement; 45, comment:# Create dict mapping subtoken->count, with additional subtokens created; 46, comment:# from substrings taken from the tokens.; 47, expression_statement; 48, comment:# Generate new list of subtokens sorted by subtoken count.; 49, expression_statement; 50, expression_statement; 51, assignment; 52, identifier:reserved_tokens; 53, call; 54, identifier:num_iterations; 55, call; 56, assignment; 57, assignment; 58, assignment; 59, call; 60, identifier:reserved_tokens; 61, identifier:RESERVED_TOKENS; 62, identifier:list; 63, argument_list; 64, attribute; 65, argument_list; 66, identifier:subtoken_dict; 67, call; 68, identifier:subtoken_counts; 69, call; 70, pattern_list; 71, call; 72, attribute; 73, argument_list; 74, identifier:alphabet; 75, attribute; 76, identifier:info; 77, binary_operator:"\tGenerating subtokens: iteration %d" % i; 78, identifier:_list_to_index_dict; 79, argument_list; 80, identifier:_count_and_gen_subtokens; 81, argument_list; 82, identifier:subtoken_list; 83, identifier:max_subtoken_length; 84, identifier:_gen_new_subtoken_list; 85, argument_list; 86, attribute; 87, identifier:info; 88, binary_operator:"\tVocab size: %d" % len(subtoken_list); 89, identifier:tf; 90, identifier:logging; 91, string:"\tGenerating subtokens: iteration %d"; 92, identifier:i; 93, identifier:subtoken_list; 94, identifier:token_counts; 95, identifier:alphabet; 96, identifier:subtoken_dict; 97, identifier:max_subtoken_length; 98, identifier:subtoken_counts; 99, identifier:min_count; 100, identifier:alphabet; 101, identifier:reserved_tokens; 102, identifier:tf; 103, identifier:logging; 104, string:"\tVocab size: %d"; 105, call; 106, identifier:len; 107, argument_list; 108, identifier:subtoken_list
code: def _generate_subtokens(
      token_counts, alphabet, min_count, num_iterations=4,
      reserved_tokens=None):
    """Create a list of subtokens in decreasing order of frequency.
  
    Args:
      token_counts: dict mapping str tokens -> int count
      alphabet: set of characters
      min_count: int minimum number of times a subtoken must appear before it is
        added to the vocabulary.
      num_iterations: int number of iterations to generate new tokens.
      reserved_tokens: list of tokens that will be added to the beginning to the
        returned subtoken list.
  
    Returns:
      Sorted list of subtokens (most frequent first)
    """
    if reserved_tokens is None:
      reserved_tokens = RESERVED_TOKENS
  
    # Use alphabet set to create initial list of subtokens
    subtoken_list = reserved_tokens + list(alphabet)
    max_subtoken_length = 1
  
    # On each iteration, segment all words using the subtokens defined in
    # subtoken_dict, count how often the resulting subtokens appear, and update
    # the dictionary with subtokens w/ high enough counts.
    for i in xrange(num_iterations):
      tf.logging.info("\tGenerating subtokens: iteration %d" % i)
      # Generate new subtoken->id dictionary using the new subtoken list.
      subtoken_dict = _list_to_index_dict(subtoken_list)
  
      # Create dict mapping subtoken->count, with additional subtokens created
      # from substrings taken from the tokens.
      subtoken_counts = _count_and_gen_subtokens(
          token_counts, alphabet, subtoken_dict, max_subtoken_length)
  
      # Generate new list of subtokens sorted by subtoken count.
      subtoken_list, max_subtoken_length = _gen_new_subtoken_list(
          subtoken_counts, min_count, alphabet, reserved_tokens)
  
      tf.logging.info("\tVocab size: %d" % len(subtoken_list))
    return subtoken_list