code: def numericalize_tok(tokens, max_vocab=50000, min_freq=0, unk_tok="_unk_", pad_tok="_pad_", bos_tok="_bos_", eos_tok="_eos_"):
      if isinstance(tokens, str):
          raise ValueError("Expected to receive a list of tokens. Received a string instead")
      if isinstance(tokens[0], list):
          tokens = [p for o in tokens for p in o]
      freq = Counter(tokens)
      int2tok = [o for o,c in freq.most_common(max_vocab) if c>min_freq]
      unk_id = 3
      int2tok.insert(0, bos_tok)
      int2tok.insert(1, pad_tok)
      int2tok.insert(2, eos_tok)
      int2tok.insert(unk_id, unk_tok)
      tok2int = collections.defaultdict(lambda:unk_id, {v:k for k,v in enumerate(int2tok)})
      return int2tok, tok2int