'''simple docstring'''






import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


SCREAMING_SNAKE_CASE_	   =			logging.get_logger(__name__)


SCREAMING_SNAKE_CASE_	   =			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all LED models at https://huggingface.co/models?filter=LED
SCREAMING_SNAKE_CASE_	   =			{
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

SCREAMING_SNAKE_CASE_	   =			{
    'allenai/led-base-16384': 1_63_84,
}






@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def  UpperCamelCase__					(					)							->	Optional[Any]:
  __UpperCAmelCase:					Optional[Any]		= (
      list(range(ord("""!"""     )   ,    ord("""~"""     ) + 1     )     ) + list(range(ord("""¡"""     )   ,    ord("""¬"""     ) + 1     )     ) + list(range(ord("""®"""     )   ,    ord("""ÿ"""     ) + 1     )     )
  )
  __UpperCAmelCase:					Optional[Any]		= bs[:]
  __UpperCAmelCase:					Tuple		= 0
  for b in range(2**8     ):
    if b not in bs:
      bs.append(_lowercase     )
      cs.append(2**8 + n     )
      n += 1
  __UpperCAmelCase:					Union[str, Any]		= [chr(_lowercase     ) for n in cs]
  return dict(zip(_lowercase   ,    _lowercase     )     )






def  UpperCamelCase__					(					_lowercase    :				int     )							->	str:
  __UpperCAmelCase:					int		= set()
  __UpperCAmelCase:					Optional[Any]		= word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)     )
    __UpperCAmelCase:					List[str]		= char
  return pairs







class    a (       __lowerCAmelCase				):



  """simple docstring"""
  __lowerCAmelCase				      =					VOCAB_FILES_NAMES
  __lowerCAmelCase				      =					PRETRAINED_VOCAB_FILES_MAP
  __lowerCAmelCase				      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowerCAmelCase				      =					["""input_ids""", """attention_mask"""]







  def __init__(							self   ,    snake_case_   ,    snake_case_   ,    snake_case_="replace"   ,    snake_case_="<s>"   ,    snake_case_="</s>"   ,    snake_case_="</s>"   ,    snake_case_="<s>"   ,    snake_case_="<unk>"   ,    snake_case_="<pad>"   ,    snake_case_="<mask>"   ,    snake_case_=False   ,    **snake_case_   ,    ):

    '''simple docstring'''







    __UpperCAmelCase:					List[str]		= AddedToken(snake_case_   ,    lstrip=snake_case_   ,    rstrip=snake_case_					) if isinstance(snake_case_   ,    snake_case_					) else bos_token
    __UpperCAmelCase:					Optional[int]		= AddedToken(snake_case_   ,    lstrip=snake_case_   ,    rstrip=snake_case_					) if isinstance(snake_case_   ,    snake_case_					) else eos_token
    __UpperCAmelCase:					int		= AddedToken(snake_case_   ,    lstrip=snake_case_   ,    rstrip=snake_case_					) if isinstance(snake_case_   ,    snake_case_					) else sep_token
    __UpperCAmelCase:					Union[str, Any]		= AddedToken(snake_case_   ,    lstrip=snake_case_   ,    rstrip=snake_case_					) if isinstance(snake_case_   ,    snake_case_					) else cls_token
    __UpperCAmelCase:					Dict		= AddedToken(snake_case_   ,    lstrip=snake_case_   ,    rstrip=snake_case_					) if isinstance(snake_case_   ,    snake_case_					) else unk_token
    __UpperCAmelCase:					Any		= AddedToken(snake_case_   ,    lstrip=snake_case_   ,    rstrip=snake_case_					) if isinstance(snake_case_   ,    snake_case_					) else pad_token

    # Mask token behave like a normal word, i.e. include the space before it
    __UpperCAmelCase:					List[Any]		= AddedToken(snake_case_   ,    lstrip=snake_case_   ,    rstrip=snake_case_					) if isinstance(snake_case_   ,    snake_case_					) else mask_token

    super().__init__(
        errors=snake_case_   ,    bos_token=snake_case_   ,    eos_token=snake_case_   ,    unk_token=snake_case_   ,    sep_token=snake_case_   ,    cls_token=snake_case_   ,    pad_token=snake_case_   ,    mask_token=snake_case_   ,    add_prefix_space=snake_case_   ,    **snake_case_   ,    )

    with open(snake_case_   ,    encoding="""utf-8"""					) as vocab_handle:
      __UpperCAmelCase:					List[str]		= json.load(snake_case_					)
    __UpperCAmelCase:					Dict		= {v: k for k, v in self.encoder.items()}
    __UpperCAmelCase:					str		= errors  # how to handle errors in decoding
    __UpperCAmelCase:					Union[str, Any]		= bytes_to_unicode()
    __UpperCAmelCase:					int		= {v: k for k, v in self.byte_encoder.items()}
    with open(snake_case_   ,    encoding="""utf-8"""					) as merges_handle:
      __UpperCAmelCase:					Optional[Any]		= merges_handle.read().split("""\n"""					)[1:-1]
    __UpperCAmelCase:					List[str]		= [tuple(merge.split()					) for merge in bpe_merges]
    __UpperCAmelCase:					List[str]		= dict(zip(snake_case_   ,    range(len(snake_case_					)					)					)					)
    __UpperCAmelCase:					List[Any]		= {}
    __UpperCAmelCase:					str		= add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    __UpperCAmelCase:					Optional[Any]		= re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""					)







  @property
  # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
  def    lowercase_  (							self					):

    '''simple docstring'''







    return len(self.encoder					)







  def    lowercase_  (							self					):

    '''simple docstring'''







    return dict(self.encoder   ,    **self.added_tokens_encoder					)







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    if token in self.cache:
      return self.cache[token]
    __UpperCAmelCase:					Any		= tuple(snake_case_					)
    __UpperCAmelCase:					Any		= get_pairs(snake_case_					)

    if not pairs:
      return token

    while True:
      __UpperCAmelCase:					List[str]		= min(snake_case_   ,    key=lambda snake_case_					: self.bpe_ranks.get(snake_case_   ,    float("""inf"""					)					)					)
      if bigram not in self.bpe_ranks:
        break
      __UpperCAmelCase, __UpperCAmelCase:					str		= bigram
      __UpperCAmelCase:					Any		= []
      __UpperCAmelCase:					Optional[Any]		= 0
      while i < len(snake_case_					):
        try:
          __UpperCAmelCase:					List[Any]		= word.index(snake_case_   ,    snake_case_					)
        except ValueError:
          new_word.extend(word[i:]					)
          break
        else:
          new_word.extend(word[i:j]					)
          __UpperCAmelCase:					Dict		= j

        if word[i] == first and i < len(snake_case_					) - 1 and word[i + 1] == second:
          new_word.append(first + second					)
          i += 2
        else:
          new_word.append(word[i]					)
          i += 1
      __UpperCAmelCase:					Optional[int]		= tuple(snake_case_					)
      __UpperCAmelCase:					Any		= new_word
      if len(snake_case_					) == 1:
        break
      else:
        __UpperCAmelCase:					int		= get_pairs(snake_case_					)
    __UpperCAmelCase:					Tuple		= """ """.join(snake_case_					)
    __UpperCAmelCase:					Union[str, Any]		= word
    return word







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					Optional[Any]		= []
    for token in re.findall(self.pat   ,    snake_case_					):
      __UpperCAmelCase:					Optional[Any]		= """""".join(
          self.byte_encoder[b] for b in token.encode("""utf-8"""					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
      bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_					).split(""" """					)					)
    return bpe_tokens







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    return self.encoder.get(snake_case_   ,    self.encoder.get(self.unk_token					)					)







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    return self.decoder.get(snake_case_					)







  def    lowercase_  (							self   ,    snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					Optional[int]		= """""".join(snake_case_					)
    __UpperCAmelCase:					Optional[int]		= bytearray([self.byte_decoder[c] for c in text]					).decode("""utf-8"""   ,    errors=self.errors					)
    return text







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

    '''simple docstring'''







    if not os.path.isdir(snake_case_					):
      logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
      return
    __UpperCAmelCase:					List[str]		= os.path.join(
        snake_case_   ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
    __UpperCAmelCase:					int		= os.path.join(
        snake_case_   ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]					)

    with open(snake_case_   ,    """w"""   ,    encoding="""utf-8"""					) as f:
      f.write(json.dumps(self.encoder   ,    indent=2   ,    sort_keys=snake_case_   ,    ensure_ascii=snake_case_					) + """\n"""					)

    __UpperCAmelCase:					Union[str, Any]		= 0
    with open(snake_case_   ,    """w"""   ,    encoding="""utf-8"""					) as writer:
      writer.write("""#version: 0.2\n"""					)
      for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,    key=lambda snake_case_					: kv[1]					):
        if index != token_index:
          logger.warning(
              F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
              """ Please check that the tokenizer is not corrupted!"""					)
          __UpperCAmelCase:					Dict		= token_index
        writer.write(""" """.join(snake_case_					) + """\n"""					)
        index += 1

    return vocab_file, merge_file







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

    '''simple docstring'''







    if token_ids_a is None:
      return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    __UpperCAmelCase:					Union[str, Any]		= [self.cls_token_id]
    __UpperCAmelCase:					Tuple		= [self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None   ,    snake_case_ = False					):

    '''simple docstring'''







    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=snake_case_   ,    token_ids_a=snake_case_   ,    already_has_special_tokens=snake_case_					)

    if token_ids_a is None:
      return [1] + ([0] * len(snake_case_					)) + [1]
    return [1] + ([0] * len(snake_case_					)) + [1, 1] + ([0] * len(snake_case_					)) + [1]







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None					):

    '''simple docstring'''







    __UpperCAmelCase:					List[str]		= [self.sep_token_id]
    __UpperCAmelCase:					Optional[Any]		= [self.cls_token_id]

    if token_ids_a is None:
      return len(cls + token_ids_a + sep					) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_=False   ,    **snake_case_					):

    '''simple docstring'''







    __UpperCAmelCase:					List[Any]		= kwargs.pop("""add_prefix_space"""   ,    self.add_prefix_space					)
    if (is_split_into_words or add_prefix_space) and (len(snake_case_					) > 0 and not text[0].isspace()):
      __UpperCAmelCase:					Optional[Any]		= """ """ + text
    return (text, kwargs)







  def    lowercase_  (							self   ,    snake_case_   ,    snake_case_ = None   ,    snake_case_ = PaddingStrategy.DO_NOT_PAD   ,    snake_case_ = None   ,    snake_case_ = None   ,    ):

    '''simple docstring'''







    __UpperCAmelCase:					Union[str, Any]		= super()._pad(
        encoded_inputs=snake_case_   ,    max_length=snake_case_   ,    padding_strategy=snake_case_   ,    pad_to_multiple_of=snake_case_   ,    return_attention_mask=snake_case_   ,    )

    # Load from model defaults
    if return_attention_mask is None:
      __UpperCAmelCase:					int		= """attention_mask""" in self.model_input_names

    if return_attention_mask and "global_attention_mask" in encoded_inputs:
      __UpperCAmelCase:					Optional[int]		= encoded_inputs[self.model_input_names[0]]
      # `global_attention_mask` need to have the same length as other (sequential) inputs.
      __UpperCAmelCase:					List[Any]		= len(encoded_inputs["""global_attention_mask"""]					) != len(snake_case_					)

      if needs_to_be_padded:
        __UpperCAmelCase:					int		= len(snake_case_					) - len(encoded_inputs["""global_attention_mask"""]					)

        if self.padding_side == "right":
          # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
          __UpperCAmelCase:					Dict		= (
              encoded_inputs["""global_attention_mask"""] + [-1] * difference
          )
        elif self.padding_side == "left":
          __UpperCAmelCase:					Union[str, Any]		= [-1] * difference + encoded_inputs[
              """global_attention_mask"""
          ]
        else:
          raise ValueError("""Invalid padding strategy:""" + str(self.padding_side					)					)

    return encoded_inputs
'''simple docstring'''






import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch




class    a (       unittest.TestCase				):



  """simple docstring"""
  __lowerCAmelCase				      =					JukeboxTokenizer
  __lowerCAmelCase				      =					{
      """artist""": """Zac Brown Band""",
      """genres""": """Country""",
      """lyrics""": """I met a traveller from an antique land,
        Who said \"Two vast and trunkless legs of stone
        Stand in the desert. . . . Near them, on the sand,
        Half sunk a shattered visage lies, whose frown,
        And wrinkled lip, and sneer of cold command,
        Tell that its sculptor well those passions read
        Which yet survive, stamped on these lifeless things,
        The hand that mocked them, and the heart that fed;
        And on the pedestal, these words appear:
        My name is Ozymandias, King of Kings;
        Look on my Works, ye Mighty, and despair!
        Nothing beside remains. Round the decay
        Of that colossal Wreck, boundless and bare
        The lone and level sands stretch far away
        """,
  }







  @require_torch
  def    lowercase_  (							self					):

    '''simple docstring'''







    import torch

    __UpperCAmelCase:					Dict		= JukeboxTokenizer.from_pretrained("""openai/jukebox-1b-lyrics"""					)
    __UpperCAmelCase:					List[str]		= tokenizer(**self.metas					)["""input_ids"""]
    # fmt: off
    __UpperCAmelCase:					int		= [
        torch.tensor([[
            0, 0, 0, 7169, 507, 9, 76, 39, 31, 46, 76, 27,
            76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
            44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
            47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
            76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
            30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
            27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
            45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
            41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
            19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
            76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
            76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
            64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
            30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
            27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
            34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
            27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
            41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
            76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
            44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
            76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
            32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
            40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
            20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
            45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
            31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
            45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
            76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
            34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
            31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
            40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
            38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
            78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
            76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
            41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
            27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
            46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
            76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
            41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
            46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
            41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
            78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
            40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
            27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
            76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
            76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
            41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
            76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
            27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
            78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
            34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
            44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
            40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
            78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
            46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
            38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
            40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
            27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
            20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
            76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
            76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
            76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
            76, 76]]					),
        torch.tensor([[0, 0, 0, 1069, 11]]					),
        torch.tensor([[0, 0, 0, 1069, 11]]					),
    ]
    # fmt: on
    self.assertTrue(torch.allclose(tokens[0]   ,    EXPECTED_OUTPUT[0]					)					)
    self.assertTrue(torch.allclose(tokens[1]   ,    EXPECTED_OUTPUT[1]					)					)
    self.assertTrue(torch.allclose(tokens[2]   ,    EXPECTED_OUTPUT[2]					)					)







  @require_torch
  def    lowercase_  (							self					):

    '''simple docstring'''







    import torch

    __UpperCAmelCase:					Any		= JukeboxTokenizer.from_pretrained("""openai/jukebox-5b-lyrics"""					)
    __UpperCAmelCase:					Tuple		= tokenizer(**self.metas					)["""input_ids"""]
    # fmt: off
    __UpperCAmelCase:					int		= [
        torch.tensor([[
            0, 0, 0, 1069, 11, -1, -1, -1, -1, 9, 77, 39,
            31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
            31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
            40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
            79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
            77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
            27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
            37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
            32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
            77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
            77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
            77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
            46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
            77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
            77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
            77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
            77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
            64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
            40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
            40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
            38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
            31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
            41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
            77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
            46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
            41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
            31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
            31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
            23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
            44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
            31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
            38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
            40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
            77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
            27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
            31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
            34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
            31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
            1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
            31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
            45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
            31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
            77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
            15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
            11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
            45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
            41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
            44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
            46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
            27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
            77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
            35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
            77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
            31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
            77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
            41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
            77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
            40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
            77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
            77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
            27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
            77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
            77, 77, 77, 77, 77, 77]]					),
        torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]					),
        torch.tensor([[0, 0, 0, 1069, 11, -1, -1, -1, -1]]					),
    ]
    # fmt: on
    self.assertTrue(torch.allclose(tokens[0]   ,    EXPECTED_OUTPUT[0]					)					)
    self.assertTrue(torch.allclose(tokens[1]   ,    EXPECTED_OUTPUT[1]					)					)
    self.assertTrue(torch.allclose(tokens[2]   ,    EXPECTED_OUTPUT[2]					)					)