nodes: 0, module; 1, function_definition; 2, function_name:numericalize_tok; 3, parameters; 4, block; 5, identifier:tokens; 6, default_parameter; 7, default_parameter; 8, default_parameter; 9, default_parameter; 10, default_parameter; 11, default_parameter; 12, expression_statement; 13, if_statement; 14, if_statement; 15, expression_statement; 16, expression_statement; 17, expression_statement; 18, expression_statement; 19, expression_statement; 20, expression_statement; 21, expression_statement; 22, expression_statement; 23, return_statement; 24, identifier:max_vocab; 25, integer:50000; 26, identifier:min_freq; 27, integer:0; 28, identifier:unk_tok; 29, string:"_unk_"; 30, identifier:pad_tok; 31, string:"_pad_"; 32, identifier:bos_tok; 33, string:"_bos_"; 34, identifier:eos_tok; 35, string:"_eos_"; 36, comment:"""Takes in text tokens and returns int2tok and tok2int converters
  
          Arguments:
          tokens(list): List of tokens. Can be a list of strings, or a list of lists of strings.
          max_vocab(int): Number of tokens to return in the vocab (sorted by frequency)
          min_freq(int): Minimum number of instances a token must be present in order to be preserved.
          unk_tok(str): Token to use when unknown tokens are encountered in the source text.
          pad_tok(str): Token to use when padding sequences.
      """; 37, call; 38, block; 39, call; 40, block; 41, assignment; 42, assignment; 43, assignment; 44, call; 45, call; 46, call; 47, call; 48, assignment; 49, expression_list; 50, identifier:isinstance; 51, argument_list; 52, raise_statement; 53, identifier:isinstance; 54, argument_list; 55, expression_statement; 56, identifier:freq; 57, call; 58, identifier:int2tok; 59, list_comprehension; 60, identifier:unk_id; 61, integer:3; 62, attribute; 63, argument_list; 64, attribute; 65, argument_list; 66, attribute; 67, argument_list; 68, attribute; 69, argument_list; 70, identifier:tok2int; 71, call; 72, identifier:int2tok; 73, identifier:tok2int; 74, identifier:tokens; 75, identifier:str; 76, call; 77, subscript; 78, identifier:list; 79, assignment; 80, identifier:Counter; 81, argument_list; 82, identifier:o; 83, for_in_clause; 84, if_clause; 85, identifier:int2tok; 86, identifier:insert; 87, integer:0; 88, identifier:bos_tok; 89, identifier:int2tok; 90, identifier:insert; 91, integer:1; 92, identifier:pad_tok; 93, identifier:int2tok; 94, identifier:insert; 95, integer:2; 96, identifier:eos_tok; 97, identifier:int2tok; 98, identifier:insert; 99, identifier:unk_id; 100, identifier:unk_tok; 101, attribute; 102, argument_list; 103, identifier:ValueError; 104, argument_list; 105, identifier:tokens; 106, integer:0; 107, identifier:tokens; 108, list_comprehension; 109, identifier:tokens; 110, pattern_list; 111, call; 112, comparison_operator:c>min_freq; 113, identifier:collections; 114, identifier:defaultdict; 115, lambda; 116, dictionary_comprehension; 117, string:"Expected to receive a list of tokens. Received a string instead"; 118, identifier:p; 119, for_in_clause; 120, for_in_clause; 121, identifier:o; 122, identifier:c; 123, attribute; 124, argument_list; 125, identifier:c; 126, identifier:min_freq; 127, identifier:unk_id; 128, pair; 129, for_in_clause; 130, identifier:o; 131, identifier:tokens; 132, identifier:p; 133, identifier:o; 134, identifier:freq; 135, identifier:most_common; 136, identifier:max_vocab; 137, identifier:v; 138, identifier:k; 139, pattern_list; 140, call; 141, identifier:k; 142, identifier:v; 143, identifier:enumerate; 144, argument_list; 145, identifier:int2tok
code: def numericalize_tok(tokens, max_vocab=50000, min_freq=0, unk_tok="_unk_", pad_tok="_pad_", bos_tok="_bos_", eos_tok="_eos_"):
      """Takes in text tokens and returns int2tok and tok2int converters
  
          Arguments:
          tokens(list): List of tokens. Can be a list of strings, or a list of lists of strings.
          max_vocab(int): Number of tokens to return in the vocab (sorted by frequency)
          min_freq(int): Minimum number of instances a token must be present in order to be preserved.
          unk_tok(str): Token to use when unknown tokens are encountered in the source text.
          pad_tok(str): Token to use when padding sequences.
      """
      if isinstance(tokens, str):
          raise ValueError("Expected to receive a list of tokens. Received a string instead")
      if isinstance(tokens[0], list):
          tokens = [p for o in tokens for p in o]
      freq = Counter(tokens)
      int2tok = [o for o,c in freq.most_common(max_vocab) if c>min_freq]
      unk_id = 3
      int2tok.insert(0, bos_tok)
      int2tok.insert(1, pad_tok)
      int2tok.insert(2, eos_tok)
      int2tok.insert(unk_id, unk_tok)
      tok2int = collections.defaultdict(lambda:unk_id, {v:k for k,v in enumerate(int2tok)})
      return int2tok, tok2int