


"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase      :Optional[Any]	    =		logging.get_logger(__name__)


__lowerCamelCase      :int	    =		{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

__lowerCamelCase      :Dict	    =		{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
}

__lowerCamelCase      :Any	    =		{
    'allenai/longformer-base-4096': 4_096,
    'allenai/longformer-large-4096': 4_096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4_096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4_096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4_096,
}




@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def   snake_case		(     )   ->      List[str]:
      lowerCamelCase					:					Optional[Any]        =	(
          list(range(ord("""!"""    )			,				ord("""~"""    ) + 1    )    ) + list(range(ord("""¡"""    )			,				ord("""¬"""    ) + 1    )    ) + list(range(ord("""®"""    )			,				ord("""ÿ"""    ) + 1    )    )
      )
      lowerCamelCase					:					Any        =	bs[:]
      lowerCamelCase					:					Optional[int]        =	0
      for b in range(2**8    ):
            if b not in bs:
                  bs.append(UpperCamelCase__    )
                  cs.append(2**8 + n    )
                  n += 1
      lowerCamelCase					:					Tuple        =	[chr(UpperCamelCase__    ) for n in cs]
      return dict(zip(UpperCamelCase__			,				UpperCamelCase__    )    )




def   snake_case		(     UpperCamelCase__       :						Tuple    )   ->      Optional[Any]:
      lowerCamelCase					:					Tuple        =	set()
      lowerCamelCase					:					Optional[Any]        =	word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)    )
            lowerCamelCase					:					List[Any]        =	char
      return pairs






class 			A__	(						__lowercase):


     """simple docstring"""
     snake_case__	:					Dict													=VOCAB_FILES_NAMES
     snake_case__	:					Dict													=PRETRAINED_VOCAB_FILES_MAP
     snake_case__	:					List[str]													=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     snake_case__	:					Optional[Any]													=['''input_ids''', '''attention_mask''']

     def __init__(	self:						Optional[Any]     ,		__a:						Optional[Any]     ,		__a:						List[Any]     ,		__a:						Dict="replace"     ,		__a:						Optional[Any]="<s>"     ,		__a:						List[Any]="</s>"     ,		__a:						List[str]="</s>"     ,		__a:						List[str]="<s>"     ,		__a:						int="<unk>"     ,		__a:						Optional[int]="<pad>"     ,		__a:						Tuple="<mask>"     ,		__a:						Optional[Any]=False     ,		**__a:						Dict     ,		)->    List[str]:
           lowerCamelCase					:					List[str]        =	AddedToken(__a     ,		lstrip=__a     ,		rstrip=__a	) if isinstance(__a     ,		__a	) else bos_token
           lowerCamelCase					:					Union[str, Any]        =	AddedToken(__a     ,		lstrip=__a     ,		rstrip=__a	) if isinstance(__a     ,		__a	) else eos_token
           lowerCamelCase					:					Optional[int]        =	AddedToken(__a     ,		lstrip=__a     ,		rstrip=__a	) if isinstance(__a     ,		__a	) else sep_token
           lowerCamelCase					:					List[str]        =	AddedToken(__a     ,		lstrip=__a     ,		rstrip=__a	) if isinstance(__a     ,		__a	) else cls_token
           lowerCamelCase					:					Any        =	AddedToken(__a     ,		lstrip=__a     ,		rstrip=__a	) if isinstance(__a     ,		__a	) else unk_token
           lowerCamelCase					:					Union[str, Any]        =	AddedToken(__a     ,		lstrip=__a     ,		rstrip=__a	) if isinstance(__a     ,		__a	) else pad_token

           # Mask token behave like a normal word, i.e. include the space before it
           lowerCamelCase					:					Union[str, Any]        =	AddedToken(__a     ,		lstrip=__a     ,		rstrip=__a	) if isinstance(__a     ,		__a	) else mask_token

           super().__init__(
               errors=__a     ,		bos_token=__a     ,		eos_token=__a     ,		unk_token=__a     ,		sep_token=__a     ,		cls_token=__a     ,		pad_token=__a     ,		mask_token=__a     ,		add_prefix_space=__a     ,		**__a     ,		)

           with open(__a     ,		encoding="""utf-8"""	) as vocab_handle:
                 lowerCamelCase					:					Optional[int]        =	json.load(__a	)
           lowerCamelCase					:					int        =	{v: k for k, v in self.encoder.items()}
           lowerCamelCase					:					Union[str, Any]        =	errors  # how to handle errors in decoding
           lowerCamelCase					:					Any        =	bytes_to_unicode()
           lowerCamelCase					:					int        =	{v: k for k, v in self.byte_encoder.items()}
           with open(__a     ,		encoding="""utf-8"""	) as merges_handle:
                 lowerCamelCase					:					Any        =	merges_handle.read().split("""\n"""	)[1:-1]
           lowerCamelCase					:					Dict        =	[tuple(merge.split()	) for merge in bpe_merges]
           lowerCamelCase					:					Dict        =	dict(zip(__a     ,		range(len(__a	)	)	)	)
           lowerCamelCase					:					Optional[int]        =	{}
           lowerCamelCase					:					Tuple        =	add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           lowerCamelCase					:					Dict        =	re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""	)

     @property
     def 		a__						(	self:						Union[str, Any]	)->    List[Any]:
           return len(self.encoder	)

     def 		a__						(	self:						int	)->    Any:
           return dict(self.encoder     ,		**self.added_tokens_encoder	)

     def 		a__						(	self:						str     ,		__a:						List[Any]	)->    Union[str, Any]:
           if token in self.cache:
                 return self.cache[token]
           lowerCamelCase					:					Any        =	tuple(__a	)
           lowerCamelCase					:					Dict        =	get_pairs(__a	)

           if not pairs:
                 return token

           while True:
                 lowerCamelCase					:					List[str]        =	min(__a     ,		key=lambda __a	: self.bpe_ranks.get(__a     ,		float("""inf"""	)	)	)
                 if bigram not in self.bpe_ranks:
                       break
                 lowerCamelCase					:					Optional[Any]        =	bigram
                 lowerCamelCase					:					Union[str, Any]        =	[]
                 lowerCamelCase					:					Optional[Any]        =	0
                 while i < len(__a	):
                       try:
                             lowerCamelCase					:					Optional[Any]        =	word.index(__a     ,		__a	)
                       except ValueError:
                             new_word.extend(word[i:]	)
                             break
                       else:
                             new_word.extend(word[i:j]	)
                             lowerCamelCase					:					Any        =	j

                       if word[i] == first and i < len(__a	) - 1 and word[i + 1] == second:
                             new_word.append(first + second	)
                             i += 2
                       else:
                             new_word.append(word[i]	)
                             i += 1
                 lowerCamelCase					:					Optional[int]        =	tuple(__a	)
                 lowerCamelCase					:					List[str]        =	new_word
                 if len(__a	) == 1:
                       break
                 else:
                       lowerCamelCase					:					Union[str, Any]        =	get_pairs(__a	)
           lowerCamelCase					:					Union[str, Any]        =	""" """.join(__a	)
           lowerCamelCase					:					Dict        =	word
           return word

     def 		a__						(	self:						str     ,		__a:						Union[str, Any]	)->    Any:
           lowerCamelCase					:					int        =	[]
           for token in re.findall(self.pat     ,		__a	):
                 lowerCamelCase					:					str        =	"""""".join(
                     self.byte_encoder[b] for b in token.encode("""utf-8"""	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                 bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__a	).split(""" """	)	)
           return bpe_tokens

     def 		a__						(	self:						Optional[Any]     ,		__a:						List[str]	)->    Optional[Any]:
           return self.encoder.get(__a     ,		self.encoder.get(self.unk_token	)	)

     def 		a__						(	self:						Union[str, Any]     ,		__a:						List[Any]	)->    Tuple:
           return self.decoder.get(__a	)

     def 		a__						(	self:						Any     ,		__a:						List[str]	)->    Any:
           lowerCamelCase					:					Union[str, Any]        =	"""""".join(__a	)
           lowerCamelCase					:					Optional[int]        =	bytearray([self.byte_decoder[c] for c in text]	).decode("""utf-8"""     ,		errors=self.errors	)
           return text

     def 		a__						(	self:						List[Any]     ,		__a:						str     ,		__a:						Optional[str] = None	)->    Tuple[str]:
           if not os.path.isdir(__a	):
                 logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
                 return
           lowerCamelCase					:					int        =	os.path.join(
               __a     ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)
           lowerCamelCase					:					Optional[int]        =	os.path.join(
               __a     ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]	)

           with open(__a     ,		"""w"""     ,		encoding="""utf-8"""	) as f:
                 f.write(json.dumps(self.encoder     ,		indent=2     ,		sort_keys=__a     ,		ensure_ascii=__a	) + """\n"""	)

           lowerCamelCase					:					List[Any]        =	0
           with open(__a     ,		"""w"""     ,		encoding="""utf-8"""	) as writer:
                 writer.write("""#version: 0.2\n"""	)
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,		key=lambda __a	: kv[1]	):
                       if index != token_index:
                             logger.warning(
                                 f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                                 """ Please check that the tokenizer is not corrupted!"""	)
                             lowerCamelCase					:					Tuple        =	token_index
                       writer.write(""" """.join(__a	) + """\n"""	)
                       index += 1

           return vocab_file, merge_file

     def 		a__						(	self:						List[Any]     ,		__a:						List[int]     ,		__a:						Optional[List[int]] = None	)->    List[int]:
           if token_ids_a is None:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           lowerCamelCase					:					Any        =	[self.cls_token_id]
           lowerCamelCase					:					Dict        =	[self.sep_token_id]
           return cls + token_ids_a + sep + sep + token_ids_a + sep

     def 		a__						(	self:						Any     ,		__a:						List[int]     ,		__a:						Optional[List[int]] = None     ,		__a:						bool = False	)->    List[int]:
           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__a     ,		token_ids_a=__a     ,		already_has_special_tokens=__a	)

           if token_ids_a is None:
                 return [1] + ([0] * len(__a	)) + [1]
           return [1] + ([0] * len(__a	)) + [1, 1] + ([0] * len(__a	)) + [1]

     def 		a__						(	self:						Tuple     ,		__a:						List[int]     ,		__a:						Optional[List[int]] = None	)->    List[int]:
           lowerCamelCase					:					int        =	[self.sep_token_id]
           lowerCamelCase					:					List[str]        =	[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep	) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]







     def 		a__						(	self:						Optional[Any]     ,		__a:						Tuple     ,		__a:						int=False     ,		**__a:						Any	)->    Tuple:
           lowerCamelCase					:					Optional[Any]        =	kwargs.pop("""add_prefix_space"""     ,		self.add_prefix_space	)
           if (is_split_into_words or add_prefix_space) and (len(__a	) > 0 and not text[0].isspace()):
                 lowerCamelCase					:					str        =	""" """ + text
           return (text, kwargs)








"""simple docstring"""


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase      :str	    =		get_tests_dir('fixtures/spiece.model')


@require_sentencepiece
@require_tokenizers
class 			A__	(						__lowercase ,     unittest.TestCase):


     """simple docstring"""
     snake_case__	:					List[str]													=AlbertTokenizer
     snake_case__	:					Optional[Any]													=AlbertTokenizerFast
     snake_case__	:					Optional[int]													=True
     snake_case__	:					Any													=True
     snake_case__	:					Optional[int]													=True

     def 		a__						(	self:						Dict	)->    Optional[Any]:
           super().setUp()

           # We have a SentencePiece fixture for testing
           lowerCamelCase					:					int        =	AlbertTokenizer(__a	)
           tokenizer.save_pretrained(self.tmpdirname	)

     def 		a__						(	self:						Tuple     ,		__a:						Tuple	)->    Union[str, Any]:
           lowerCamelCase					:					List[str]        =	"""this is a test"""
           lowerCamelCase					:					int        =	"""this is a test"""
           return input_text, output_text

     def 		a__						(	self:						Any	)->    List[Any]:
           lowerCamelCase					:					int        =	"""<pad>"""
           lowerCamelCase					:					Optional[int]        =	0

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a	)     ,		__a	)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a	)     ,		__a	)

     def 		a__						(	self:						Tuple	)->    str:
           lowerCamelCase					:					Optional[int]        =	list(self.get_tokenizer().get_vocab().keys()	)

           self.assertEqual(vocab_keys[0]     ,		"""<pad>"""	)
           self.assertEqual(vocab_keys[1]     ,		"""<unk>"""	)
           self.assertEqual(vocab_keys[-1]     ,		"""▁eloquent"""	)
           self.assertEqual(len(__a	)     ,		30_000	)

     def 		a__						(	self:						List[str]	)->    Any:
           self.assertEqual(self.get_tokenizer().vocab_size     ,		30_000	)

     def 		a__						(	self:						Optional[Any]	)->    Union[str, Any]:
           if not self.test_rust_tokenizer:
                 return

           lowerCamelCase					:					str        =	self.get_tokenizer()
           lowerCamelCase					:					Tuple        =	self.get_rust_tokenizer()

           lowerCamelCase					:					Union[str, Any]        =	"""I was born in 92000, and this is falsé."""

           lowerCamelCase					:					List[str]        =	tokenizer.tokenize(__a	)
           lowerCamelCase					:					Tuple        =	rust_tokenizer.tokenize(__a	)
           self.assertListEqual(__a     ,		__a	)

           lowerCamelCase					:					Dict        =	tokenizer.encode(__a     ,		add_special_tokens=__a	)
           lowerCamelCase					:					List[str]        =	rust_tokenizer.encode(__a     ,		add_special_tokens=__a	)
           self.assertListEqual(__a     ,		__a	)

           lowerCamelCase					:					Any        =	self.get_rust_tokenizer()
           lowerCamelCase					:					List[str]        =	tokenizer.encode(__a	)
           lowerCamelCase					:					str        =	rust_tokenizer.encode(__a	)
           self.assertListEqual(__a     ,		__a	)

     def 		a__						(	self:						Tuple	)->    List[Any]:
           lowerCamelCase					:					List[str]        =	AlbertTokenizer(__a     ,		keep_accents=__a	)

           lowerCamelCase					:					int        =	tokenizer.tokenize("""This is a test"""	)
           self.assertListEqual(__a     ,		["""▁this""", """▁is""", """▁a""", """▁test"""]	)

           self.assertListEqual(tokenizer.convert_tokens_to_ids(__a	)     ,		[48, 25, 21, 1_289]	)

           lowerCamelCase					:					List[str]        =	tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
           self.assertListEqual(
               __a     ,		["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """."""]	)
           lowerCamelCase					:					List[str]        =	tokenizer.convert_tokens_to_ids(__a	)
           self.assertListEqual(__a     ,		[31, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]	)

           lowerCamelCase					:					Tuple        =	tokenizer.convert_ids_to_tokens(__a	)
           self.assertListEqual(
               __a     ,		["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """."""]     ,		)

     def 		a__						(	self:						Tuple	)->    str:
           lowerCamelCase					:					str        =	AlbertTokenizer(__a	)

           lowerCamelCase					:					Union[str, Any]        =	tokenizer.encode("""sequence builders"""	)
           lowerCamelCase					:					List[Any]        =	tokenizer.encode("""multi-sequence build"""	)

           lowerCamelCase					:					Any        =	tokenizer.build_inputs_with_special_tokens(__a	)
           lowerCamelCase					:					str        =	tokenizer.build_inputs_with_special_tokens(__a     ,		__a	)

           assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
           assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
               tokenizer.sep_token_id
           ]







     @slow
     def 		a__						(	self:						Any	)->    Dict:
           # fmt: off
           lowerCamelCase					:					Optional[Any]        =	{"""attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """input_ids""": [[2, 21_970, 13, 5, 6_092, 167, 28, 7_103, 2_153, 673, 8, 7_028, 12_051, 18, 17, 7_103, 2_153, 673, 8, 3_515, 18_684, 8, 4_461, 6, 1_927, 297, 8, 12_060, 2_607, 18, 13, 5, 4_461, 15, 10_538, 38, 8, 135, 15, 822, 58, 15, 993, 10_363, 15, 1_460, 8_005, 4_461, 15, 993, 255, 2_328, 9, 9, 9, 6, 26, 1_112, 816, 3_260, 13, 5, 103, 2_377, 6, 17, 1_112, 816, 2_782, 13, 5, 103, 10_641, 6, 29, 84, 2_512, 2_430, 782, 18_684, 2_761, 19, 808, 2_430, 2_556, 17, 855, 1_480, 9_477, 4_091, 128, 11_712, 15, 7_103, 2_153, 673, 17, 24_883, 9_990, 9, 3], [2, 11_502, 25, 1_006, 20, 782, 8, 11_809, 855, 1_732, 19_393, 18_667, 37, 367, 21_018, 69, 1_854, 34, 11_860, 19_124, 27, 156, 225, 17, 193, 4_141, 19, 65, 9_124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2_231, 886, 2_385, 17_659, 84, 14, 16_792, 1_952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=__a     ,		model_name="""albert-base-v2"""     ,		revision="""6b6560eaf5ff2e250b00c50f380c5389a9c2d82e"""     ,		)




