



'''simple docstring'''



import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
						from ...tokenization_utils_base import TextInput
from ...utils import logging


lowerCAmelCase	:		Optional[int]				=   logging.get_logger(__name__)

lowerCAmelCase	:		Union[str, Any]				=   {'''vocab_file''': '''spiece.model'''}

lowerCAmelCase	:		Optional[Any]				=   {
    '''vocab_file''': {
        '''t5-small''': '''https://huggingface.co/t5-small/resolve/main/spiece.model''',
        '''t5-base''': '''https://huggingface.co/t5-base/resolve/main/spiece.model''',
        '''t5-large''': '''https://huggingface.co/t5-large/resolve/main/spiece.model''',
        '''t5-3b''': '''https://huggingface.co/t5-3b/resolve/main/spiece.model''',
        '''t5-11b''': '''https://huggingface.co/t5-11b/resolve/main/spiece.model''',
    }
}


# TODO(PVP) - this should be removed in Transformers v5
lowerCAmelCase	:		Optional[Any]				=   {
    '''t5-small''': 5_1_2,
    '''t5-base''': 5_1_2,
    '''t5-large''': 5_1_2,
    '''t5-3b''': 5_1_2,
    '''t5-11b''': 5_1_2,
}

lowerCAmelCase	:		str				=   '''▁'''






class 					UpperCAmelCase__	(		__A ):
					a      :     int        =							VOCAB_FILES_NAMES
					a      :     str        =							PRETRAINED_VOCAB_FILES_MAP
					a      :     List[str]        =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					a      :     List[Any]        =							["""input_ids""", """attention_mask"""]



					def __init__(    self			,				UpperCamelCase			,				UpperCamelCase="</s>"			,				UpperCamelCase="<unk>"			,				UpperCamelCase="<pad>"			,				UpperCamelCase=100			,				UpperCamelCase=None			,				UpperCamelCase = None			,				UpperCamelCase=True			,				**UpperCamelCase			,				)     ->			List[Any]:
						# Add extra_ids to the special token list
						if extra_ids > 0 and additional_special_tokens is None:
							__lowerCAmelCase												=  [F'''<extra_id_{i}>''' for i in range(UpperCamelCase__     )]
						elif extra_ids > 0 and additional_special_tokens is not None:
							# Check that we have the right number of extra_id special tokens
							__lowerCAmelCase												=  len(set(filter(lambda UpperCamelCase     : bool("extra_id" in str(UpperCamelCase__     )     )			,				UpperCamelCase__     )     )     )
							if extra_tokens != extra_ids:
								raise ValueError(
								    F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
								    " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
								    " tokens"     )
						if legacy:
							logger.warning_once(
							    F'''You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'''
							    " read the related pull request available at https://github.com/huggingface/transformers/pull/24565"     )

						__lowerCAmelCase												=  legacy
						__lowerCAmelCase												=  {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    eos_token=UpperCamelCase__			,				unk_token=UpperCamelCase__			,				pad_token=UpperCamelCase__			,				extra_ids=UpperCamelCase__			,				additional_special_tokens=UpperCamelCase__			,				sp_model_kwargs=self.sp_model_kwargs			,				legacy=UpperCamelCase__			,				**UpperCamelCase__			,				)

						__lowerCAmelCase												=  vocab_file
						__lowerCAmelCase												=  extra_ids

						__lowerCAmelCase												=  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.Load(UpperCamelCase__     )



					@staticmethod
					def        UpperCAmelCase_ (    UpperCamelCase			,				UpperCamelCase			,				UpperCamelCase     )     ->			List[Any]:
						if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
							__lowerCAmelCase												=  TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
							if init_max_model_length is not None and init_max_model_length != max_model_length:
								return init_max_model_length
							elif init_max_model_length is None:
								warnings.warn(
								    "This tokenizer was incorrectly instantiated with a model max length of"
								    F''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
								    " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
								    " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
								    F''' {pretrained_model_name_or_path} automatically truncating your input to'''
								    F''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
								    F''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
								    " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
								    " instantiate this tokenizer with `model_max_length` set to your preferred value."			,				UpperCamelCase__			,				)

						return max_model_length



					@property
					def        UpperCAmelCase_ (    self     )     ->			List[Any]:
						return self.sp_model.get_piece_size() + self._extra_ids



					def        UpperCAmelCase_ (    self     )     ->			Optional[Any]:
						__lowerCAmelCase												=  {self.convert_ids_to_tokens(UpperCamelCase__     ): i for i in range(self.vocab_size     )}
						vocab.update(self.added_tokens_encoder     )
						return vocab



					def        UpperCAmelCase_ (    self			,				UpperCamelCase			,				UpperCamelCase = None			,				UpperCamelCase = False     )     ->			Optional[int]:
						if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=UpperCamelCase__			,				token_ids_a=UpperCamelCase__			,				already_has_special_tokens=UpperCamelCase__     )

						# normal case: some special tokens
						if token_ids_a is None:
							return ([0] * len(UpperCamelCase__     )) + [1]
						return ([0] * len(UpperCamelCase__     )) + [1] + ([0] * len(UpperCamelCase__     )) + [1]



					def        UpperCAmelCase_ (    self     )     ->			List[str]:
						return list(
						    set(filter(lambda UpperCamelCase     : bool(re.search(r"<extra_id_\d+>"			,				UpperCamelCase__     )     ) is not None			,				self.additional_special_tokens     )     )     )



					def        UpperCAmelCase_ (    self     )     ->			Optional[int]:
						return [self._convert_token_to_id(UpperCamelCase__     ) for token in self.get_sentinel_tokens()]



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			int:
						if len(UpperCamelCase__     ) > 0 and token_ids[-1] == self.eos_token_id:
							warnings.warn(
							    F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
							    " eos tokens being added."     )
							return token_ids
						else:
							return token_ids + [self.eos_token_id]



					def        UpperCAmelCase_ (    self			,				UpperCamelCase			,				UpperCamelCase = None     )     ->			List[str]:
						__lowerCAmelCase												=  [self.eos_token_id]

						if token_ids_a is None:
							return len(token_ids_a + eos     ) * [0]
						return len(token_ids_a + eos + token_ids_a + eos     ) * [0]



					def        UpperCAmelCase_ (    self			,				UpperCamelCase			,				UpperCamelCase = None     )     ->			List[Any]:
						__lowerCAmelCase												=  self._add_eos_if_not_present(UpperCamelCase__     )
						if token_ids_a is None:
							return token_ids_a
						else:
							__lowerCAmelCase												=  self._add_eos_if_not_present(UpperCamelCase__     )
							return token_ids_a + token_ids_a



					def __getstate__(    self     )     ->			str:
						__lowerCAmelCase												=  self.__dict__.copy()
						__lowerCAmelCase												=  None
						return state



					def __setstate__(    self			,				UpperCamelCase     )     ->			List[str]:
						__lowerCAmelCase												=  d

						# for backward compatibility
						if not hasattr(self			,				"sp_model_kwargs"     ):
							__lowerCAmelCase												=  {}

						__lowerCAmelCase												=  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
						self.sp_model.Load(self.vocab_file     )



					def        UpperCAmelCase_ (    self			,				UpperCamelCase			,				**UpperCamelCase     )     ->			List[str]:
						# Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
						# the beginning of the text
						if not self.legacy:
							__lowerCAmelCase												=  SPIECE_UNDERLINE + text.replace(UpperCamelCase__			,				" "     )
						return super().tokenize(UpperCamelCase__			,				**UpperCamelCase__     )



					def        UpperCAmelCase_ (    self			,				UpperCamelCase			,				**UpperCamelCase     )     ->			Union[str, Any]:
						if not self.legacy:
							__lowerCAmelCase												=  text.startswith(UpperCamelCase__     )
							if is_first:
								__lowerCAmelCase												=  text[1:]

						__lowerCAmelCase												=  self.sp_model.encode(UpperCamelCase__			,				out_type=UpperCamelCase__     )

						if not self.legacy and not is_first and not text.startswith(" "     ) and tokens[0].startswith(UpperCamelCase__     ):
							__lowerCAmelCase												=  ([tokens[0][1:]] if len(tokens[0]     ) > 1 else []) + tokens[1:]
						return tokens



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			List[str]:
						if token.startswith("<extra_id_"     ):
							__lowerCAmelCase												=  re.match(r"<extra_id_(\d+)>"			,				UpperCamelCase__     )
							__lowerCAmelCase												=  int(match.group(1     )     )
							return self.vocab_size - num - 1
						return self.sp_model.piece_to_id(UpperCamelCase__     )



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			str:
						if index < self.sp_model.get_piece_size():
							__lowerCAmelCase												=  self.sp_model.IdToPiece(UpperCamelCase__     )
						else:
							__lowerCAmelCase												=  F'''<extra_id_{self.vocab_size - 1 - index}>'''
						return token



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			Dict:
						__lowerCAmelCase												=  []
						__lowerCAmelCase												=  ''
						__lowerCAmelCase												=  False
						for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
								if not prev_is_special:
									out_string += " "
								out_string += self.sp_model.decode(UpperCamelCase__     ) + token
								__lowerCAmelCase												=  True
								__lowerCAmelCase												=  []
							else:
								current_sub_tokens.append(UpperCamelCase__     )
								__lowerCAmelCase												=  False
						out_string += self.sp_model.decode(UpperCamelCase__     )
						return out_string.strip()



					def        UpperCAmelCase_ (    self			,				UpperCamelCase			,				UpperCamelCase = None     )     ->			Any:
						if not os.path.isdir(UpperCamelCase__     ):
							logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
							return
						__lowerCAmelCase												=  os.path.join(
						    UpperCamelCase__			,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

						if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase__     ) and os.path.isfile(self.vocab_file     ):
							copyfile(self.vocab_file			,				UpperCamelCase__     )
						elif not os.path.isfile(self.vocab_file     ):
							with open(UpperCamelCase__			,				"wb"     ) as fi:
								__lowerCAmelCase												=  self.sp_model.serialized_model_proto()
								fi.write(UpperCamelCase__     )

						return (out_vocab_file,)




'''simple docstring'''



from abc import ABC, abstractmethod
from typing import List, Optional






class 					UpperCAmelCase__	(		UpperCamelCase__ ):



					def __init__(    self     )     ->			List[str]:
						# test for the above condition
						self.test()



					def        UpperCAmelCase_ (    self     )     ->			Dict:
						__lowerCAmelCase												=  0
						__lowerCAmelCase												=  False
						while not completed:
							if counter == 1:
								self.reset()
							__lowerCAmelCase												=  self.advance()
							if not self.does_advance(UpperCamelCase     ):
								raise Exception(
								    "Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true."     )

							__lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase												=  self.update(UpperCamelCase     )
							counter += 1

							if counter > 1_0000:
								raise Exception("update() does not fulfill the constraint."     )

						if self.remaining() != 0:
							raise Exception("Custom Constraint is not defined correctly."     )



					@abstractmethod
					def        UpperCAmelCase_ (    self     )     ->			Dict:
						raise NotImplementedError(
						    F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



					@abstractmethod
					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			Optional[int]:
						raise NotImplementedError(
						    F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



					@abstractmethod
					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			Any:
						raise NotImplementedError(
						    F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



					@abstractmethod
					def        UpperCAmelCase_ (    self     )     ->			int:
						raise NotImplementedError(
						    F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



					@abstractmethod
					def        UpperCAmelCase_ (    self     )     ->			int:
						raise NotImplementedError(
						    F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )



					@abstractmethod
					def        UpperCAmelCase_ (    self			,				UpperCamelCase=False     )     ->			str:
						raise NotImplementedError(
						    F'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.'''     )






class 					UpperCAmelCase__	(		UpperCamelCase__ ):



					def __init__(    self			,				UpperCamelCase     )     ->			Dict:
						super(UpperCamelCase			,				self     ).__init__()

						if not isinstance(UpperCamelCase			,				UpperCamelCase     ) or len(UpperCamelCase     ) == 0:
							raise ValueError(F'''`token_ids` has to be a non-empty list, but is {token_ids}.'''     )
						if any((not isinstance(UpperCamelCase			,				UpperCamelCase     ) or token_id < 0) for token_id in token_ids     ):
							raise ValueError(F'''Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.'''     )

						__lowerCAmelCase												=  token_ids

						__lowerCAmelCase												=  len(self.token_ids     )
						__lowerCAmelCase												=  -1  # the index of the currently fulfilled step
						__lowerCAmelCase												=  False



					def        UpperCAmelCase_ (    self     )     ->			Optional[int]:
						if self.completed:
							return None
						return self.token_ids[self.fulfilled_idx + 1]



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			str:
						if not isinstance(UpperCamelCase			,				UpperCamelCase     ):
							raise ValueError(F'''`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase     )}'''     )

						if self.completed:
							return False

						return token_id == self.token_ids[self.fulfilled_idx + 1]



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			int:
						if not isinstance(UpperCamelCase			,				UpperCamelCase     ):
							raise ValueError(F'''`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase     )}'''     )

						__lowerCAmelCase												=  False
						__lowerCAmelCase												=  False
						__lowerCAmelCase												=  False

						if self.does_advance(UpperCamelCase     ):
							self.fulfilled_idx += 1
							__lowerCAmelCase												=  True
							if self.fulfilled_idx == (self.seqlen - 1):
								__lowerCAmelCase												=  True
							__lowerCAmelCase												=  completed
						else:
							# failed to make progress.
							__lowerCAmelCase												=  True
							self.reset()
						return stepped, completed, reset



					def        UpperCAmelCase_ (    self     )     ->			Union[str, Any]:
						__lowerCAmelCase												=  False
						__lowerCAmelCase												=  0



					def        UpperCAmelCase_ (    self     )     ->			Optional[int]:
						return self.seqlen - (self.fulfilled_idx + 1)



					def        UpperCAmelCase_ (    self			,				UpperCamelCase=False     )     ->			Optional[Any]:
						__lowerCAmelCase												=  PhrasalConstraint(self.token_ids     )

						if stateful:
							__lowerCAmelCase												=  self.seqlen
							__lowerCAmelCase												=  self.fulfilled_idx
							__lowerCAmelCase												=  self.completed

						return new_constraint






class 					UpperCAmelCase__	:



					def __init__(    self			,				UpperCamelCase			,				UpperCamelCase=True     )     ->			Optional[int]:
						__lowerCAmelCase												=  max([len(UpperCamelCase     ) for one in nested_token_ids]     )

						__lowerCAmelCase												=  {}
						for token_ids in nested_token_ids:
							__lowerCAmelCase												=  root
							for tidx, token_id in enumerate(UpperCamelCase     ):
								if token_id not in level:
									__lowerCAmelCase												=  {}

								__lowerCAmelCase												=  level[token_id]

						if no_subsets and self.has_subsets(UpperCamelCase			,				UpperCamelCase     ):
							raise ValueError(
							    "Each list in `nested_token_ids` can't be a complete subset of another list, but is"
							    F''' {nested_token_ids}.'''     )

						__lowerCAmelCase												=  root



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			int:
						__lowerCAmelCase												=  self.trie

						for current_token in current_seq:
							__lowerCAmelCase												=  start[current_token]

						__lowerCAmelCase												=  list(start.keys()     )

						return next_tokens



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			str:
						__lowerCAmelCase												=  self.next_tokens(UpperCamelCase     )

						return len(UpperCamelCase     ) == 0



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			Optional[int]:
						__lowerCAmelCase												=  list(root.values()     )
						if len(UpperCamelCase     ) == 0:
							return 1
						else:
							return sum([self.count_leaves(UpperCamelCase     ) for nn in next_nodes]     )



					def        UpperCAmelCase_ (    self			,				UpperCamelCase			,				UpperCamelCase     )     ->			Optional[Any]:
						__lowerCAmelCase												=  self.count_leaves(UpperCamelCase     )
						return len(UpperCamelCase     ) != leaf_count






class 					UpperCAmelCase__	(		UpperCamelCase__ ):



					def __init__(    self			,				UpperCamelCase     )     ->			List[Any]:
						super(UpperCamelCase			,				self     ).__init__()

						if not isinstance(UpperCamelCase			,				UpperCamelCase     ) or len(UpperCamelCase     ) == 0:
							raise ValueError(F'''`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.'''     )
						if any(not isinstance(UpperCamelCase			,				UpperCamelCase     ) for token_ids in nested_token_ids     ):
							raise ValueError(F'''`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.'''     )
						if any(
						    any((not isinstance(UpperCamelCase			,				UpperCamelCase     ) or token_id < 0) for token_id in token_ids     )
						    for token_ids in nested_token_ids     ):
							raise ValueError(
							    F'''Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}.'''     )

						__lowerCAmelCase												=  DisjunctiveTrie(UpperCamelCase     )
						__lowerCAmelCase												=  nested_token_ids

						__lowerCAmelCase												=  self.trie.max_height
						__lowerCAmelCase												=  []
						__lowerCAmelCase												=  False



					def        UpperCAmelCase_ (    self     )     ->			List[Any]:
						__lowerCAmelCase												=  self.trie.next_tokens(self.current_seq     )

						if len(UpperCamelCase     ) == 0:
							return None
						else:
							return token_list



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			List[str]:
						if not isinstance(UpperCamelCase			,				UpperCamelCase     ):
							raise ValueError(F'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase     )}'''     )

						__lowerCAmelCase												=  self.trie.next_tokens(self.current_seq     )

						return token_id in next_tokens



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			int:
						if not isinstance(UpperCamelCase			,				UpperCamelCase     ):
							raise ValueError(F'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase     )}'''     )

						__lowerCAmelCase												=  False
						__lowerCAmelCase												=  False
						__lowerCAmelCase												=  False

						if self.does_advance(UpperCamelCase     ):
							self.current_seq.append(UpperCamelCase     )
							__lowerCAmelCase												=  True
						else:
							__lowerCAmelCase												=  True
							self.reset()

						__lowerCAmelCase												=  self.trie.reached_leaf(self.current_seq     )
						__lowerCAmelCase												=  completed

						return stepped, completed, reset



					def        UpperCAmelCase_ (    self     )     ->			Dict:
						__lowerCAmelCase												=  False
						__lowerCAmelCase												=  []



					def        UpperCAmelCase_ (    self     )     ->			int:
						if self.completed:
							# since this can be completed without reaching max height
							return 0
						else:
							return self.seqlen - len(self.current_seq     )



					def        UpperCAmelCase_ (    self			,				UpperCamelCase=False     )     ->			Union[str, Any]:
						__lowerCAmelCase												=  DisjunctiveConstraint(self.token_ids     )

						if stateful:
							__lowerCAmelCase												=  self.seqlen
							__lowerCAmelCase												=  self.current_seq
							__lowerCAmelCase												=  self.completed

						return new_constraint






class 					UpperCAmelCase__	:



					def __init__(    self			,				UpperCamelCase     )     ->			Union[str, Any]:
						__lowerCAmelCase												=  constraints

						# max # of steps required to fulfill a given constraint
						__lowerCAmelCase												=  max([c.seqlen for c in constraints]     )
						__lowerCAmelCase												=  len(UpperCamelCase     )
						__lowerCAmelCase												=  False

						self.init_state()



					def        UpperCAmelCase_ (    self     )     ->			Union[str, Any]:
						__lowerCAmelCase												=  []
						__lowerCAmelCase												=  None
						__lowerCAmelCase												=  [constraint.copy(stateful=UpperCamelCase     ) for constraint in self.constraints]



					def        UpperCAmelCase_ (    self     )     ->			Optional[int]:
						__lowerCAmelCase												=  0
						if self.inprogress_constraint:
							# extra points for having a constraint mid-fulfilled
							add += self.max_seqlen - self.inprogress_constraint.remaining()

						return (len(self.complete_constraints     ) * self.max_seqlen) + add



					def        UpperCAmelCase_ (    self     )     ->			List[str]:
						__lowerCAmelCase												=  []
						if self.inprogress_constraint is None:
							for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
								__lowerCAmelCase												=  constraint.advance()
								if isinstance(UpperCamelCase			,				UpperCamelCase     ):
									token_list.append(UpperCamelCase     )
								elif isinstance(UpperCamelCase			,				UpperCamelCase     ):
									token_list.extend(UpperCamelCase     )
						else:
							__lowerCAmelCase												=  self.inprogress_constraint.advance()
							if isinstance(UpperCamelCase			,				UpperCamelCase     ):
								token_list.append(UpperCamelCase     )
							elif isinstance(UpperCamelCase			,				UpperCamelCase     ):
								token_list.extend(UpperCamelCase     )

						if len(UpperCamelCase     ) == 0:
							return None
						else:
							return token_list



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			int:
						self.init_state()

						if token_ids is not None:
							for token in token_ids:
								# completes or steps **one** constraint
								__lowerCAmelCase							,       __lowerCAmelCase												=  self.add(UpperCamelCase     )

								# the entire list of constraints are fulfilled
								if self.completed:
									break



					def        UpperCAmelCase_ (    self			,				UpperCamelCase     )     ->			Dict:
						if not isinstance(UpperCamelCase			,				UpperCamelCase     ):
							raise ValueError(F'''`token_id` should be an `int`, but is `{token_id}`.'''     )

						__lowerCAmelCase							,       __lowerCAmelCase												=  False, False

						if self.completed:
							__lowerCAmelCase												=  True
							__lowerCAmelCase												=  False
							return complete, stepped

						if self.inprogress_constraint is not None:
							# In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
							# job, simply update the state

							__lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase												=  self.inprogress_constraint.update(UpperCamelCase     )
							if reset:
								# 1. If the next token breaks the progress, then we must restart.
								#     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

								#     But that doesn't mean we self.init_state(), since we only reset the state for this particular
								#     constraint, not the full list of constraints.

								self.pending_constraints.append(self.inprogress_constraint.copy(stateful=UpperCamelCase     )     )
								__lowerCAmelCase												=  None

							if complete:
								# 2. If the next token completes the constraint, move it to completed list, set
								#     inprogress to None. If there are no pending constraints either, then this full list of constraints
								#     is complete.

								self.complete_constraints.append(self.inprogress_constraint     )
								__lowerCAmelCase												=  None

								if len(self.pending_constraints     ) == 0:
									# we're done!
									__lowerCAmelCase												=  True

						else:
							# Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
							# of constraints?

							for cidx, pending_constraint in enumerate(self.pending_constraints     ):
								if pending_constraint.does_advance(UpperCamelCase     ):
									__lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase												=  pending_constraint.update(UpperCamelCase     )

									if not stepped:
										raise Exception(
										    "`constraint.update(token_id)` is not yielding incremental progress, "
										    "even though `constraint.does_advance(token_id)` is true."     )

									if complete:
										self.complete_constraints.append(UpperCamelCase     )
										__lowerCAmelCase												=  None

									if not complete and stepped:
										__lowerCAmelCase												=  pending_constraint

									if complete or stepped:
										# If we made any progress at all, then it's at least not a "pending constraint".

										__lowerCAmelCase												=  (
										    self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
										)

										if len(self.pending_constraints     ) == 0 and self.inprogress_constraint is None:
											# If there's no longer any pending after this and no inprogress either, then we must be
											# complete.

											__lowerCAmelCase												=  True

										break  # prevent accidentally stepping through multiple constraints with just one token.

						return complete, stepped



					def        UpperCAmelCase_ (    self			,				UpperCamelCase=True     )     ->			str:
						__lowerCAmelCase												=  ConstraintListState(self.constraints     )  # we actually never though self.constraints objects
						# throughout this process. So it's at initialization state.

						if stateful:
							__lowerCAmelCase												=  [
							    constraint.copy(stateful=UpperCamelCase     ) for constraint in self.complete_constraints
							]
							if self.inprogress_constraint is not None:
								__lowerCAmelCase												=  self.inprogress_constraint.copy(stateful=UpperCamelCase     )
							__lowerCAmelCase												=  [constraint.copy() for constraint in self.pending_constraints]

						return new_state