




from abc import ABC, abstractmethod
from typing import List, Optional







class 		lowercase							(     snake_case__	):

     def __init__(						self       )		->							Tuple:
           self.test()

     def       a__			(						self       )		->							str:
           _A							:  Any    =		0
           _A							:  Dict    =		False
           while not completed:
                 if counter == 1:
                       self.reset()
                 _A							:  Optional[int]    =		self.advance()
                 if not self.does_advance(UpperCAmelCase_       ):
                       raise Exception(
                           """Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true."""       )

                 _A    ,     _A    ,     _A							:  str    =		self.update(UpperCAmelCase_       )
                 counter += 1

                 if counter > 1_0000:
                       raise Exception("""update() does not fulfill the constraint."""       )

           if self.remaining() != 0:
                 raise Exception("""Custom Constraint is not defined correctly."""       )

     @abstractmethod
     def       a__			(						self       )		->							Optional[Any]:
           raise NotImplementedError(
               F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''       )

     @abstractmethod
     def       a__			(						self	,  _a       )		->							Tuple:
           raise NotImplementedError(
               F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''       )

     @abstractmethod
     def       a__			(						self	,  _a       )		->							List[str]:
           raise NotImplementedError(
               F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''       )

     @abstractmethod
     def       a__			(						self       )		->							int:
           raise NotImplementedError(
               F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''       )

     @abstractmethod
     def       a__			(						self       )		->							List[str]:
           raise NotImplementedError(
               F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''       )



     @abstractmethod
     def       a__			(						self	,  _a=False       )		->							List[Any]:
           raise NotImplementedError(
               F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''       )







class 		lowercase							(     snake_case__	):

     def __init__(						self	,  _a       )		->							str:
           super(UpperCAmelCase_	,  self       ).__init__()

           if not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ) or len(UpperCAmelCase_       ) == 0:
                 raise ValueError(F'''`token_ids` has to be a non-empty list, but is {token_ids}.'''       )
           if any((not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ) or token_id < 0) for token_id in token_ids       ):
                 raise ValueError(F'''Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.'''       )

           _A							:  Dict    =		token_ids

           _A							:  Optional[int]    =		len(self.token_ids       )
           _A							:  List[str]    =		-1  # the index of the currently fulfilled step
           _A							:  Tuple    =		False

     def       a__			(						self       )		->							Dict:



           if self.completed:
                 return None
           return self.token_ids[self.fulfilled_idx + 1]

     def       a__			(						self	,  _a       )		->							Optional[int]:



           if not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ):
                 raise ValueError(F'''`token_id` has to be an `int`, but is {token_id} of type {type(UpperCAmelCase_       )}'''       )

           if self.completed:
                 return False

           return token_id == self.token_ids[self.fulfilled_idx + 1]

     def       a__			(						self	,  _a       )		->							List[Any]:



           if not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ):
                 raise ValueError(F'''`token_id` has to be an `int`, but is {token_id} of type {type(UpperCAmelCase_       )}'''       )

           _A							:  Any    =		False
           _A							:  List[str]    =		False
           _A							:  List[str]    =		False

           if self.does_advance(UpperCAmelCase_       ):
                 self.fulfilled_idx += 1
                 _A							:  Dict    =		True
                 if self.fulfilled_idx == (self.seqlen - 1):
                       _A							:  Any    =		True
                 _A							:  Any    =		completed
           else:
                 # failed to make progress.
                 _A							:  str    =		True
                 self.reset()
           return stepped, completed, reset

     def       a__			(						self       )		->							Optional[Any]:
           _A							:  Dict    =		False
           _A							:  Optional[Any]    =		0

     def       a__			(						self       )		->							Dict:
           return self.seqlen - (self.fulfilled_idx + 1)



     def       a__			(						self	,  _a=False       )		->							List[str]:
           _A							:  Optional[Any]    =		PhrasalConstraint(self.token_ids       )

           if stateful:
                 _A							:  List[Any]    =		self.seqlen
                 _A							:  int    =		self.fulfilled_idx
                 _A							:  Any    =		self.completed

           return new_constraint







class 		lowercase							:

     def __init__(						self	,  _a	,  _a=True       )		->							Union[str, Any]:
           _A							:  Any    =		max([len(UpperCAmelCase_       ) for one in nested_token_ids]       )

           _A							:  Optional[Any]    =		{}
           for token_ids in nested_token_ids:
                 _A							:  List[str]    =		root
                 for tidx, token_id in enumerate(UpperCAmelCase_       ):
                       if token_id not in level:
                             _A							:  Optional[int]    =		{}

                       _A							:  Optional[Any]    =		level[token_id]

           if no_subsets and self.has_subsets(UpperCAmelCase_	,  UpperCAmelCase_       ):
                 raise ValueError(
                     """Each list in `nested_token_ids` can't be a complete subset of another list, but is"""
                     F''' {nested_token_ids}.'''       )

           _A							:  Any    =		root

     def       a__			(						self	,  _a       )		->							Optional[Any]:
           _A							:  int    =		self.trie

           for current_token in current_seq:
                 _A							:  Dict    =		start[current_token]

           _A							:  str    =		list(start.keys()       )

           return next_tokens

     def       a__			(						self	,  _a       )		->							Optional[int]:
           _A							:  Optional[Any]    =		self.next_tokens(UpperCAmelCase_       )

           return len(UpperCAmelCase_       ) == 0

     def       a__			(						self	,  _a       )		->							List[str]:
           _A							:  List[Any]    =		list(root.values()       )
           if len(UpperCAmelCase_       ) == 0:
                 return 1
           else:
                 return sum([self.count_leaves(UpperCAmelCase_       ) for nn in next_nodes]       )



     def       a__			(						self	,  _a	,  _a       )		->							List[Any]:
           _A							:  List[str]    =		self.count_leaves(UpperCAmelCase_       )
           return len(UpperCAmelCase_       ) != leaf_count







class 		lowercase							(     snake_case__	):

     def __init__(						self	,  _a       )		->							List[str]:
           super(UpperCAmelCase_	,  self       ).__init__()

           if not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ) or len(UpperCAmelCase_       ) == 0:
                 raise ValueError(F'''`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.'''       )
           if any(not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ) for token_ids in nested_token_ids       ):
                 raise ValueError(F'''`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.'''       )
           if any(
               any((not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ) or token_id < 0) for token_id in token_ids       )
               for token_ids in nested_token_ids       ):
                 raise ValueError(
                     F'''Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}.'''       )

           _A							:  List[Any]    =		DisjunctiveTrie(UpperCAmelCase_       )
           _A							:  Union[str, Any]    =		nested_token_ids

           _A							:  int    =		self.trie.max_height
           _A							:  List[Any]    =		[]
           _A							:  Optional[int]    =		False

     def       a__			(						self       )		->							List[Any]:
           _A							:  Dict    =		self.trie.next_tokens(self.current_seq       )

           if len(UpperCAmelCase_       ) == 0:
                 return None
           else:
                 return token_list

     def       a__			(						self	,  _a       )		->							int:



           if not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ):
                 raise ValueError(F'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCAmelCase_       )}'''       )

           _A							:  Tuple    =		self.trie.next_tokens(self.current_seq       )

           return token_id in next_tokens

     def       a__			(						self	,  _a       )		->							Dict:



           if not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ):
                 raise ValueError(F'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCAmelCase_       )}'''       )

           _A							:  str    =		False
           _A							:  str    =		False
           _A							:  Optional[int]    =		False

           if self.does_advance(UpperCAmelCase_       ):
                 self.current_seq.append(UpperCAmelCase_       )
                 _A							:  List[str]    =		True
           else:
                 _A							:  Optional[int]    =		True
                 self.reset()

           _A							:  Optional[Any]    =		self.trie.reached_leaf(self.current_seq       )
           _A							:  Any    =		completed

           return stepped, completed, reset

     def       a__			(						self       )		->							Dict:
           _A							:  Optional[int]    =		False
           _A							:  List[Any]    =		[]

     def       a__			(						self       )		->							int:



           if self.completed:
                 # since this can be completed without reaching max height
                 return 0
           else:
                 return self.seqlen - len(self.current_seq       )



     def       a__			(						self	,  _a=False       )		->							List[Any]:
           _A							:  Tuple    =		DisjunctiveConstraint(self.token_ids       )

           if stateful:
                 _A							:  Optional[int]    =		self.seqlen
                 _A							:  List[Any]    =		self.current_seq
                 _A							:  List[str]    =		self.completed

           return new_constraint







class 		lowercase							:

     def __init__(						self	,  _a       )		->							Any:
           _A							:  int    =		constraints

           # max # of steps required to fulfill a given constraint
           _A							:  Optional[Any]    =		max([c.seqlen for c in constraints]       )
           _A							:  Optional[int]    =		len(UpperCAmelCase_       )
           _A							:  List[str]    =		False

           self.init_state()

     def       a__			(						self       )		->							Dict:
           _A							:  int    =		[]
           _A							:  List[str]    =		None
           _A							:  Tuple    =		[constraint.copy(stateful=UpperCAmelCase_       ) for constraint in self.constraints]

     def       a__			(						self       )		->							List[Any]:
           _A							:  Dict    =		0
           if self.inprogress_constraint:
                 # extra points for having a constraint mid-fulfilled
                 add += self.max_seqlen - self.inprogress_constraint.remaining()

           return (len(self.complete_constraints       ) * self.max_seqlen) + add

     def       a__			(						self       )		->							List[Any]:
           _A							:  str    =		[]
           if self.inprogress_constraint is None:
                 for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
                       _A							:  str    =		constraint.advance()
                       if isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ):
                             token_list.append(UpperCAmelCase_       )
                       elif isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ):
                             token_list.extend(UpperCAmelCase_       )
           else:
                 _A							:  str    =		self.inprogress_constraint.advance()
                 if isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ):
                       token_list.append(UpperCAmelCase_       )
                 elif isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ):
                       token_list.extend(UpperCAmelCase_       )

           if len(UpperCAmelCase_       ) == 0:
                 return None
           else:
                 return token_list

     def       a__			(						self	,  _a       )		->							Dict:
           self.init_state()

           if token_ids is not None:
                 for token in token_ids:
                       # completes or steps **one** constraint
                       _A    ,     _A							:  Any    =		self.add(UpperCAmelCase_       )

                       # the entire list of constraints are fulfilled
                       if self.completed:
                             break

     def       a__			(						self	,  _a       )		->							List[Any]:



           if not isinstance(UpperCAmelCase_	,  UpperCAmelCase_       ):
                 raise ValueError(F'''`token_id` should be an `int`, but is `{token_id}`.'''       )

           _A    ,     _A							:  Any    =		False, False

           if self.completed:
                 _A							:  int    =		True
                 _A							:  Optional[int]    =		False
                 return complete, stepped

           if self.inprogress_constraint is not None:
                 # In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
                 # job, simply update the state

                 _A    ,     _A    ,     _A							:  Union[str, Any]    =		self.inprogress_constraint.update(UpperCAmelCase_       )
                 if reset:
                       # 1. If the next token breaks the progress, then we must restart.
                       #     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

                       #     But that doesn't mean we self.init_state(), since we only reset the state for this particular
                       #     constraint, not the full list of constraints.

                       self.pending_constraints.append(self.inprogress_constraint.copy(stateful=UpperCAmelCase_       )       )
                       _A							:  Any    =		None

                 if complete:
                       # 2. If the next token completes the constraint, move it to completed list, set
                       #     inprogress to None. If there are no pending constraints either, then this full list of constraints
                       #     is complete.

                       self.complete_constraints.append(self.inprogress_constraint       )
                       _A							:  str    =		None

                       if len(self.pending_constraints       ) == 0:
                             # we're done!
                             _A							:  List[str]    =		True

           else:
                 # Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
                 # of constraints?

                 for cidx, pending_constraint in enumerate(self.pending_constraints       ):
                       if pending_constraint.does_advance(UpperCAmelCase_       ):
                             _A    ,     _A    ,     _A							:  Dict    =		pending_constraint.update(UpperCAmelCase_       )

                             if not stepped:
                                   raise Exception(
                                       """`constraint.update(token_id)` is not yielding incremental progress, """
                                       """even though `constraint.does_advance(token_id)` is true."""       )

                             if complete:
                                   self.complete_constraints.append(UpperCAmelCase_       )
                                   _A							:  Optional[Any]    =		None

                             if not complete and stepped:
                                   _A							:  int    =		pending_constraint

                             if complete or stepped:
                                   # If we made any progress at all, then it's at least not a "pending constraint".

                                   _A							:  Any    =		(
                                       self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
                                   )

                                   if len(self.pending_constraints       ) == 0 and self.inprogress_constraint is None:
                                         # If there's no longer any pending after this and no inprogress either, then we must be
                                         # complete.

                                         _A							:  Optional[int]    =		True

                                   break  # prevent accidentally stepping through multiple constraints with just one token.

           return complete, stepped



     def       a__			(						self	,  _a=True       )		->							Optional[int]:
           _A							:  Union[str, Any]    =		ConstraintListState(self.constraints       )  # we actually never though self.constraints objects
           # throughout this process. So it's at initialization state.

           if stateful:
                 _A							:  Optional[Any]    =		[
                     constraint.copy(stateful=UpperCAmelCase_       ) for constraint in self.complete_constraints
                 ]
                 if self.inprogress_constraint is not None:
                       _A							:  Any    =		self.inprogress_constraint.copy(stateful=UpperCAmelCase_       )
                 _A							:  Any    =		[constraint.copy() for constraint in self.pending_constraints]

           return new_state







import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 		lowercase							(     UpperCamelCase__,unittest.TestCase	):
     _a											=		CLIPTokenizer
     _a											=		CLIPTokenizerFast
     _a											=		True
     _a											=		{}
     _a											=		False

     def       a__			(						self       )		->							Optional[Any]:
           super().setUp()

           # fmt: off
           _A							:  int    =		["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
           # fmt: on
           _A							:  str    =		dict(zip(_a	,  range(len(_a       )       )       )       )
           _A							:  Optional[int]    =		["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
           _A							:  str    =		{"""unk_token""": """<unk>"""}

           _A							:  Union[str, Any]    =		os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["""vocab_file"""]       )
           _A							:  Union[str, Any]    =		os.path.join(self.tmpdirname	,  VOCAB_FILES_NAMES["""merges_file"""]       )
           with open(self.vocab_file	,  """w"""	,  encoding="""utf-8"""       ) as fp:
                 fp.write(json.dumps(_a       ) + """\n"""       )
           with open(self.merges_file	,  """w"""	,  encoding="""utf-8"""       ) as fp:
                 fp.write("""\n""".join(_a       )       )

     def       a__			(						self	,  **_a       )		->							List[Any]:
           kwargs.update(self.special_tokens_map       )
           return CLIPTokenizer.from_pretrained(self.tmpdirname	,  **_a       )

     def       a__			(						self	,  **_a       )		->							Union[str, Any]:
           kwargs.update(self.special_tokens_map       )
           return CLIPTokenizerFast.from_pretrained(self.tmpdirname	,  **_a       )

     def       a__			(						self	,  _a       )		->							str:
           _A							:  Tuple    =		"""lower newer"""
           _A							:  Optional[Any]    =		"""lower newer"""
           return input_text, output_text

     def       a__			(						self       )		->							List[Any]:
           _A							:  Optional[int]    =		CLIPTokenizer(self.vocab_file	,  self.merges_file	,  **self.special_tokens_map       )
           _A							:  str    =		"""lower newer"""
           _A							:  List[str]    =		["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
           _A							:  int    =		tokenizer.tokenize(_a       )
           self.assertListEqual(_a	,  _a       )

           _A							:  str    =		tokens + [tokenizer.unk_token]
           _A							:  Dict    =		[10, 2, 16, 9, 3, 2, 16, 20]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(_a       )	,  _a       )

     @require_ftfy
     def       a__			(						self       )		->							Tuple:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                       _A							:  Dict    =		self.tokenizer_class.from_pretrained(_a	,  **_a       )
                       _A							:  str    =		self.rust_tokenizer_class.from_pretrained(_a	,  **_a       )

                       _A							:  List[str]    =		"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
                       _A							:  Dict    =		tokenizer_s.tokenize(_a       )
                       _A							:  Dict    =		tokenizer_r.tokenize(_a       )

                       self.assertListEqual(_a	,  _a       )

                       # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                       # with Tilde) encoded in 2 different ways
                       _A							:  Tuple    =		"""xa\u0303y""" + """ """ + """x\xe3y"""
                       _A							:  Dict    =		tokenizer_s.tokenize(_a       )
                       _A							:  Dict    =		tokenizer_r.tokenize(_a       )

                       self.assertListEqual(_a	,  _a       )

                       # Test that the tokenization is identical on unicode of space type
                       _A							:  Any    =		[
                           """\u0009""",  # (horizontal tab, '\t')
                           """\u000B""",  # (vertical tab)
                           """\u000C""",  # (form feed)
                           """\u0020""",  # (space, ' ')
                           """\u200E""",  # (left-to-right mark):w
                           """\u200F""",  # (right-to-left mark)
                       ]
                       for unicode_seq in spaces_unicodes:
                             _A							:  Optional[int]    =		tokenizer_s.tokenize(_a       )
                             _A							:  List[Any]    =		tokenizer_r.tokenize(_a       )

                             self.assertListEqual(_a	,  _a       )

                       # Test that the tokenization is identical on unicode of line break type
                       _A							:  int    =		[
                           """\u000A""",  # (line feed, '\n')
                           """\r\n""",  # (carriage return and line feed, '\r\n')
                           """\u000D""",  # (carriage return, '\r')
                           """\r""",  # (carriage return, '\r')
                           """\u000D""",  # (carriage return, '\r')
                           """\u2028""",  # (line separator)
                           """\u2029""",  # (paragraph separator)
                           # "\u0085", # (next line)
                       ]

                       # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                       # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                       # space (and thus into an empty list).

                       for unicode_seq in line_break_unicodes:
                             _A							:  Optional[Any]    =		tokenizer_s.tokenize(_a       )
                             _A							:  Tuple    =		tokenizer_r.tokenize(_a       )

                             self.assertListEqual(_a	,  _a       )

     def       a__			(						self       )		->							Any:
           # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                       _A							:  List[Any]    =		"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
                       _A							:  str    =		F'''{text_of_1_token} {text_of_1_token}'''

                       _A							:  str    =		self.rust_tokenizer_class.from_pretrained(
                           _a	,  use_fast=_a	,  )
                       _A							:  Dict    =		tokenizer_r(_a	,  return_offsets_mapping=_a	,  add_special_tokens=_a       )
                       self.assertEqual(encoding.offset_mapping[0]	,  (0, len(_a       ))       )
                       self.assertEqual(
                           encoding.offset_mapping[1]	,  (len(_a       ) + 1, len(_a       ) + 1 + len(_a       ))	,  )

                       _A							:  Any    =		F''' {text}'''

                       _A							:  List[Any]    =		self.rust_tokenizer_class.from_pretrained(
                           _a	,  use_fast=_a	,  )
                       _A							:  Optional[int]    =		tokenizer_r(_a	,  return_offsets_mapping=_a	,  add_special_tokens=_a       )
                       self.assertEqual(encoding.offset_mapping[0]	,  (1, 1 + len(_a       ))       )
                       self.assertEqual(
                           encoding.offset_mapping[1]	,  (1 + len(_a       ) + 1, 1 + len(_a       ) + 1 + len(_a       ))	,  )

     def       a__			(						self       )		->							List[Any]:
           # Test related to the breaking change introduced in transformers v4.17.0
           # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
           with self.assertRaises(_a       ) as context:
                 self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""       )

           self.assertTrue(
               context.exception.args[0].startswith(
                   """The `backend_tokenizer` provided does not match the expected format."""       )       )

     @require_ftfy
     def       a__			(						self       )		->							str:
           super().test_tokenization_python_rust_equals()



     def       a__			(						self       )		->							Union[str, Any]:
           # CLIP always lower cases letters
           pass

