


"""simple docstring"""


from abc import ABC, abstractmethod
from typing import List, Optional


class 			A__	(						__a):


     """simple docstring"""

     def __init__(	self:						Dict	)->    List[str]:
           self.test()

     def 		a__						(	self:						Optional[Any]	)->    Dict:
           lowerCamelCase					:					Tuple        =	0
           lowerCamelCase					:					List[Any]        =	False
           while not completed:
                 if counter == 1:
                       self.reset()
                 lowerCamelCase					:					List[str]        =	self.advance()
                 if not self.does_advance(a_	):
                       raise Exception(
                           """Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true."""	)

                 lowerCamelCase					:					Union[str, Any]        =	self.update(a_	)
                 counter += 1

                 if counter > 10_000:
                       raise Exception("""update() does not fulfill the constraint."""	)

           if self.remaining() != 0:
                 raise Exception("""Custom Constraint is not defined correctly."""	)

     @abstractmethod
     def 		a__						(	self:						Optional[int]	)->    Union[str, Any]:
           raise NotImplementedError(
               f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'	)

     @abstractmethod
     def 		a__						(	self:						Optional[int]     ,		__a:						int	)->    Union[str, Any]:
           raise NotImplementedError(
               f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'	)

     @abstractmethod
     def 		a__						(	self:						Optional[int]     ,		__a:						int	)->    List[str]:
           raise NotImplementedError(
               f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'	)

     @abstractmethod
     def 		a__						(	self:						Tuple	)->    Tuple:
           raise NotImplementedError(
               f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'	)

     @abstractmethod
     def 		a__						(	self:						int	)->    int:
           raise NotImplementedError(
               f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'	)







     @abstractmethod
     def 		a__						(	self:						List[str]     ,		__a:						Tuple=False	)->    str:
           raise NotImplementedError(
               f'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'	)


class 			A__	(						__a):


     """simple docstring"""

     def __init__(	self:						Tuple     ,		__a:						List[int]	)->    List[str]:
           super(a_     ,		self	).__init__()

           if not isinstance(a_     ,		a_	) or len(a_	) == 0:
                 raise ValueError(f'`token_ids` has to be a non-empty list, but is {token_ids}.'	)
           if any((not isinstance(a_     ,		a_	) or token_id < 0) for token_id in token_ids	):
                 raise ValueError(f'Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.'	)

           lowerCamelCase					:					Optional[int]        =	token_ids

           lowerCamelCase					:					Dict        =	len(self.token_ids	)
           lowerCamelCase					:					Union[str, Any]        =	-1  # the index of the currently fulfilled step
           lowerCamelCase					:					str        =	False

     def 		a__						(	self:						Optional[int]	)->    List[str]:



           if self.completed:
                 return None
           return self.token_ids[self.fulfilled_idx + 1]

     def 		a__						(	self:						List[Any]     ,		__a:						int	)->    Optional[Any]:



           if not isinstance(a_     ,		a_	):
                 raise ValueError(f'`token_id` has to be an `int`, but is {token_id} of type {type(a_	)}'	)

           if self.completed:
                 return False

           return token_id == self.token_ids[self.fulfilled_idx + 1]

     def 		a__						(	self:						List[str]     ,		__a:						int	)->    Optional[int]:



           if not isinstance(a_     ,		a_	):
                 raise ValueError(f'`token_id` has to be an `int`, but is {token_id} of type {type(a_	)}'	)

           lowerCamelCase					:					Optional[int]        =	False
           lowerCamelCase					:					List[str]        =	False
           lowerCamelCase					:					List[Any]        =	False

           if self.does_advance(a_	):
                 self.fulfilled_idx += 1
                 lowerCamelCase					:					List[str]        =	True
                 if self.fulfilled_idx == (self.seqlen - 1):
                       lowerCamelCase					:					str        =	True
                 lowerCamelCase					:					Optional[int]        =	completed
           else:
                 # failed to make progress.
                 lowerCamelCase					:					Union[str, Any]        =	True
                 self.reset()
           return stepped, completed, reset

     def 		a__						(	self:						Union[str, Any]	)->    Optional[int]:
           lowerCamelCase					:					int        =	False
           lowerCamelCase					:					Dict        =	0

     def 		a__						(	self:						Tuple	)->    Optional[int]:
           return self.seqlen - (self.fulfilled_idx + 1)







     def 		a__						(	self:						Union[str, Any]     ,		__a:						str=False	)->    Union[str, Any]:
           lowerCamelCase					:					Dict        =	PhrasalConstraint(self.token_ids	)

           if stateful:
                 lowerCamelCase					:					List[Any]        =	self.seqlen
                 lowerCamelCase					:					str        =	self.fulfilled_idx
                 lowerCamelCase					:					Union[str, Any]        =	self.completed

           return new_constraint


class 			A__	:


     """simple docstring"""

     def __init__(	self:						int     ,		__a:						List[List[int]]     ,		__a:						int=True	)->    Dict:
           lowerCamelCase					:					List[str]        =	max([len(a_	) for one in nested_token_ids]	)

           lowerCamelCase					:					int        =	{}
           for token_ids in nested_token_ids:
                 lowerCamelCase					:					List[str]        =	root
                 for tidx, token_id in enumerate(a_	):
                       if token_id not in level:
                             lowerCamelCase					:					int        =	{}

                       lowerCamelCase					:					List[Any]        =	level[token_id]

           if no_subsets and self.has_subsets(a_     ,		a_	):
                 raise ValueError(
                     """Each list in `nested_token_ids` can't be a complete subset of another list, but is"""
                     f' {nested_token_ids}.'	)

           lowerCamelCase					:					int        =	root

     def 		a__						(	self:						Tuple     ,		__a:						Tuple	)->    Dict:
           lowerCamelCase					:					Union[str, Any]        =	self.trie

           for current_token in current_seq:
                 lowerCamelCase					:					Any        =	start[current_token]

           lowerCamelCase					:					Any        =	list(start.keys()	)

           return next_tokens

     def 		a__						(	self:						List[Any]     ,		__a:						Optional[int]	)->    List[Any]:
           lowerCamelCase					:					Any        =	self.next_tokens(a_	)

           return len(a_	) == 0

     def 		a__						(	self:						Dict     ,		__a:						List[Any]	)->    Any:
           lowerCamelCase					:					int        =	list(root.values()	)
           if len(a_	) == 0:
                 return 1
           else:
                 return sum([self.count_leaves(a_	) for nn in next_nodes]	)







     def 		a__						(	self:						str     ,		__a:						List[Any]     ,		__a:						int	)->    int:
           lowerCamelCase					:					str        =	self.count_leaves(a_	)
           return len(a_	) != leaf_count


class 			A__	(						__a):


     """simple docstring"""

     def __init__(	self:						Tuple     ,		__a:						List[List[int]]	)->    int:
           super(a_     ,		self	).__init__()

           if not isinstance(a_     ,		a_	) or len(a_	) == 0:
                 raise ValueError(f'`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.'	)
           if any(not isinstance(a_     ,		a_	) for token_ids in nested_token_ids	):
                 raise ValueError(f'`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.'	)
           if any(
               any((not isinstance(a_     ,		a_	) or token_id < 0) for token_id in token_ids	)
               for token_ids in nested_token_ids	):
                 raise ValueError(
                     f'Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}.'	)

           lowerCamelCase					:					Optional[Any]        =	DisjunctiveTrie(a_	)
           lowerCamelCase					:					Optional[int]        =	nested_token_ids

           lowerCamelCase					:					Tuple        =	self.trie.max_height
           lowerCamelCase					:					List[str]        =	[]
           lowerCamelCase					:					Union[str, Any]        =	False

     def 		a__						(	self:						Any	)->    Any:
           lowerCamelCase					:					List[Any]        =	self.trie.next_tokens(self.current_seq	)

           if len(a_	) == 0:
                 return None
           else:
                 return token_list

     def 		a__						(	self:						Tuple     ,		__a:						int	)->    List[str]:



           if not isinstance(a_     ,		a_	):
                 raise ValueError(f'`token_id` is supposed to be type `int`, but is {token_id} of type {type(a_	)}'	)

           lowerCamelCase					:					Optional[Any]        =	self.trie.next_tokens(self.current_seq	)

           return token_id in next_tokens

     def 		a__						(	self:						Any     ,		__a:						int	)->    List[Any]:



           if not isinstance(a_     ,		a_	):
                 raise ValueError(f'`token_id` is supposed to be type `int`, but is {token_id} of type {type(a_	)}'	)

           lowerCamelCase					:					Any        =	False
           lowerCamelCase					:					Union[str, Any]        =	False
           lowerCamelCase					:					Tuple        =	False

           if self.does_advance(a_	):
                 self.current_seq.append(a_	)
                 lowerCamelCase					:					Optional[Any]        =	True
           else:
                 lowerCamelCase					:					Tuple        =	True
                 self.reset()

           lowerCamelCase					:					Any        =	self.trie.reached_leaf(self.current_seq	)
           lowerCamelCase					:					Tuple        =	completed

           return stepped, completed, reset

     def 		a__						(	self:						str	)->    int:
           lowerCamelCase					:					Any        =	False
           lowerCamelCase					:					Tuple        =	[]

     def 		a__						(	self:						str	)->    Union[str, Any]:



           if self.completed:
                 # since this can be completed without reaching max height
                 return 0
           else:
                 return self.seqlen - len(self.current_seq	)







     def 		a__						(	self:						Dict     ,		__a:						List[Any]=False	)->    List[str]:
           lowerCamelCase					:					Union[str, Any]        =	DisjunctiveConstraint(self.token_ids	)

           if stateful:
                 lowerCamelCase					:					Tuple        =	self.seqlen
                 lowerCamelCase					:					List[str]        =	self.current_seq
                 lowerCamelCase					:					List[Any]        =	self.completed

           return new_constraint


class 			A__	:


     """simple docstring"""

     def __init__(	self:						Any     ,		__a:						List[Constraint]	)->    Optional[Any]:
           lowerCamelCase					:					Any        =	constraints

           # max # of steps required to fulfill a given constraint
           lowerCamelCase					:					Optional[int]        =	max([c.seqlen for c in constraints]	)
           lowerCamelCase					:					Optional[Any]        =	len(a_	)
           lowerCamelCase					:					List[Any]        =	False

           self.init_state()

     def 		a__						(	self:						Optional[int]	)->    Optional[Any]:
           lowerCamelCase					:					List[Any]        =	[]
           lowerCamelCase					:					Union[str, Any]        =	None
           lowerCamelCase					:					Optional[int]        =	[constraint.copy(stateful=a_	) for constraint in self.constraints]

     def 		a__						(	self:						Union[str, Any]	)->    List[str]:
           lowerCamelCase					:					Any        =	0
           if self.inprogress_constraint:
                 # extra points for having a constraint mid-fulfilled
                 add += self.max_seqlen - self.inprogress_constraint.remaining()

           return (len(self.complete_constraints	) * self.max_seqlen) + add

     def 		a__						(	self:						Optional[int]	)->    Any:
           lowerCamelCase					:					Union[str, Any]        =	[]
           if self.inprogress_constraint is None:
                 for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
                       lowerCamelCase					:					str        =	constraint.advance()
                       if isinstance(a_     ,		a_	):
                             token_list.append(a_	)
                       elif isinstance(a_     ,		a_	):
                             token_list.extend(a_	)
           else:
                 lowerCamelCase					:					List[str]        =	self.inprogress_constraint.advance()
                 if isinstance(a_     ,		a_	):
                       token_list.append(a_	)
                 elif isinstance(a_     ,		a_	):
                       token_list.extend(a_	)

           if len(a_	) == 0:
                 return None
           else:
                 return token_list

     def 		a__						(	self:						int     ,		__a:						Optional[List[int]]	)->    List[Any]:
           self.init_state()

           if token_ids is not None:
                 for token in token_ids:
                       # completes or steps **one** constraint
                       lowerCamelCase					:					int        =	self.add(a_	)

                       # the entire list of constraints are fulfilled
                       if self.completed:
                             break

     def 		a__						(	self:						List[Any]     ,		__a:						int	)->    int:



           if not isinstance(a_     ,		a_	):
                 raise ValueError(f'`token_id` should be an `int`, but is `{token_id}`.'	)

           lowerCamelCase					:					List[str]        =	False, False

           if self.completed:
                 lowerCamelCase					:					Dict        =	True
                 lowerCamelCase					:					List[Any]        =	False
                 return complete, stepped

           if self.inprogress_constraint is not None:
                 # In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
                 # job, simply update the state

                 lowerCamelCase					:					Optional[int]        =	self.inprogress_constraint.update(a_	)
                 if reset:
                       # 1. If the next token breaks the progress, then we must restart.
                       #     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

                       #     But that doesn't mean we self.init_state(), since we only reset the state for this particular
                       #     constraint, not the full list of constraints.

                       self.pending_constraints.append(self.inprogress_constraint.copy(stateful=a_	)	)
                       lowerCamelCase					:					Union[str, Any]        =	None

                 if complete:
                       # 2. If the next token completes the constraint, move it to completed list, set
                       #     inprogress to None. If there are no pending constraints either, then this full list of constraints
                       #     is complete.

                       self.complete_constraints.append(self.inprogress_constraint	)
                       lowerCamelCase					:					Dict        =	None

                       if len(self.pending_constraints	) == 0:
                             # we're done!
                             lowerCamelCase					:					Union[str, Any]        =	True

           else:
                 # Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
                 # of constraints?

                 for cidx, pending_constraint in enumerate(self.pending_constraints	):
                       if pending_constraint.does_advance(a_	):
                             lowerCamelCase					:					Any        =	pending_constraint.update(a_	)

                             if not stepped:
                                   raise Exception(
                                       """`constraint.update(token_id)` is not yielding incremental progress, """
                                       """even though `constraint.does_advance(token_id)` is true."""	)

                             if complete:
                                   self.complete_constraints.append(a_	)
                                   lowerCamelCase					:					Optional[Any]        =	None

                             if not complete and stepped:
                                   lowerCamelCase					:					List[Any]        =	pending_constraint

                             if complete or stepped:
                                   # If we made any progress at all, then it's at least not a "pending constraint".

                                   lowerCamelCase					:					Any        =	(
                                       self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
                                   )

                                   if len(self.pending_constraints	) == 0 and self.inprogress_constraint is None:
                                         # If there's no longer any pending after this and no inprogress either, then we must be
                                         # complete.

                                         lowerCamelCase					:					List[str]        =	True

                                   break  # prevent accidentally stepping through multiple constraints with just one token.

           return complete, stepped







     def 		a__						(	self:						Dict     ,		__a:						int=True	)->    Optional[int]:
           lowerCamelCase					:					Dict        =	ConstraintListState(self.constraints	)  # we actually never though self.constraints objects
           # throughout this process. So it's at initialization state.

           if stateful:
                 lowerCamelCase					:					List[Any]        =	[
                     constraint.copy(stateful=a_	) for constraint in self.complete_constraints
                 ]
                 if self.inprogress_constraint is not None:
                       lowerCamelCase					:					Dict        =	self.inprogress_constraint.copy(stateful=a_	)
                 lowerCamelCase					:					List[str]        =	[constraint.copy() for constraint in self.pending_constraints]

           return new_state








"""simple docstring"""


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase      :str	    =		get_tests_dir('fixtures/spiece.model')


@require_sentencepiece
@require_tokenizers
class 			A__	(						__lowercase ,     unittest.TestCase):


     """simple docstring"""
     snake_case__	:					List[str]													=AlbertTokenizer
     snake_case__	:					Optional[Any]													=AlbertTokenizerFast
     snake_case__	:					Optional[int]													=True
     snake_case__	:					Any													=True
     snake_case__	:					Optional[int]													=True

     def 		a__						(	self:						Dict	)->    Optional[Any]:
           super().setUp()

           # We have a SentencePiece fixture for testing
           lowerCamelCase					:					int        =	AlbertTokenizer(__a	)
           tokenizer.save_pretrained(self.tmpdirname	)

     def 		a__						(	self:						Tuple     ,		__a:						Tuple	)->    Union[str, Any]:
           lowerCamelCase					:					List[str]        =	"""this is a test"""
           lowerCamelCase					:					int        =	"""this is a test"""
           return input_text, output_text

     def 		a__						(	self:						Any	)->    List[Any]:
           lowerCamelCase					:					int        =	"""<pad>"""
           lowerCamelCase					:					Optional[int]        =	0

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a	)     ,		__a	)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a	)     ,		__a	)

     def 		a__						(	self:						Tuple	)->    str:
           lowerCamelCase					:					Optional[int]        =	list(self.get_tokenizer().get_vocab().keys()	)

           self.assertEqual(vocab_keys[0]     ,		"""<pad>"""	)
           self.assertEqual(vocab_keys[1]     ,		"""<unk>"""	)
           self.assertEqual(vocab_keys[-1]     ,		"""▁eloquent"""	)
           self.assertEqual(len(__a	)     ,		30_000	)

     def 		a__						(	self:						List[str]	)->    Any:
           self.assertEqual(self.get_tokenizer().vocab_size     ,		30_000	)

     def 		a__						(	self:						Optional[Any]	)->    Union[str, Any]:
           if not self.test_rust_tokenizer:
                 return

           lowerCamelCase					:					str        =	self.get_tokenizer()
           lowerCamelCase					:					Tuple        =	self.get_rust_tokenizer()

           lowerCamelCase					:					Union[str, Any]        =	"""I was born in 92000, and this is falsé."""

           lowerCamelCase					:					List[str]        =	tokenizer.tokenize(__a	)
           lowerCamelCase					:					Tuple        =	rust_tokenizer.tokenize(__a	)
           self.assertListEqual(__a     ,		__a	)

           lowerCamelCase					:					Dict        =	tokenizer.encode(__a     ,		add_special_tokens=__a	)
           lowerCamelCase					:					List[str]        =	rust_tokenizer.encode(__a     ,		add_special_tokens=__a	)
           self.assertListEqual(__a     ,		__a	)

           lowerCamelCase					:					Any        =	self.get_rust_tokenizer()
           lowerCamelCase					:					List[str]        =	tokenizer.encode(__a	)
           lowerCamelCase					:					str        =	rust_tokenizer.encode(__a	)
           self.assertListEqual(__a     ,		__a	)

     def 		a__						(	self:						Tuple	)->    List[Any]:
           lowerCamelCase					:					List[str]        =	AlbertTokenizer(__a     ,		keep_accents=__a	)

           lowerCamelCase					:					int        =	tokenizer.tokenize("""This is a test"""	)
           self.assertListEqual(__a     ,		["""▁this""", """▁is""", """▁a""", """▁test"""]	)

           self.assertListEqual(tokenizer.convert_tokens_to_ids(__a	)     ,		[48, 25, 21, 1_289]	)

           lowerCamelCase					:					List[str]        =	tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
           self.assertListEqual(
               __a     ,		["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """."""]	)
           lowerCamelCase					:					List[str]        =	tokenizer.convert_tokens_to_ids(__a	)
           self.assertListEqual(__a     ,		[31, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]	)

           lowerCamelCase					:					Tuple        =	tokenizer.convert_ids_to_tokens(__a	)
           self.assertListEqual(
               __a     ,		["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """."""]     ,		)

     def 		a__						(	self:						Tuple	)->    str:
           lowerCamelCase					:					str        =	AlbertTokenizer(__a	)

           lowerCamelCase					:					Union[str, Any]        =	tokenizer.encode("""sequence builders"""	)
           lowerCamelCase					:					List[Any]        =	tokenizer.encode("""multi-sequence build"""	)

           lowerCamelCase					:					Any        =	tokenizer.build_inputs_with_special_tokens(__a	)
           lowerCamelCase					:					str        =	tokenizer.build_inputs_with_special_tokens(__a     ,		__a	)

           assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
           assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
               tokenizer.sep_token_id
           ]







     @slow
     def 		a__						(	self:						Any	)->    Dict:
           # fmt: off
           lowerCamelCase					:					Optional[Any]        =	{"""attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """input_ids""": [[2, 21_970, 13, 5, 6_092, 167, 28, 7_103, 2_153, 673, 8, 7_028, 12_051, 18, 17, 7_103, 2_153, 673, 8, 3_515, 18_684, 8, 4_461, 6, 1_927, 297, 8, 12_060, 2_607, 18, 13, 5, 4_461, 15, 10_538, 38, 8, 135, 15, 822, 58, 15, 993, 10_363, 15, 1_460, 8_005, 4_461, 15, 993, 255, 2_328, 9, 9, 9, 6, 26, 1_112, 816, 3_260, 13, 5, 103, 2_377, 6, 17, 1_112, 816, 2_782, 13, 5, 103, 10_641, 6, 29, 84, 2_512, 2_430, 782, 18_684, 2_761, 19, 808, 2_430, 2_556, 17, 855, 1_480, 9_477, 4_091, 128, 11_712, 15, 7_103, 2_153, 673, 17, 24_883, 9_990, 9, 3], [2, 11_502, 25, 1_006, 20, 782, 8, 11_809, 855, 1_732, 19_393, 18_667, 37, 367, 21_018, 69, 1_854, 34, 11_860, 19_124, 27, 156, 225, 17, 193, 4_141, 19, 65, 9_124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2_231, 886, 2_385, 17_659, 84, 14, 16_792, 1_952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=__a     ,		model_name="""albert-base-v2"""     ,		revision="""6b6560eaf5ff2e250b00c50f380c5389a9c2d82e"""     ,		)




