




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A						:							Optional[int]							=  logging.get_logger(__name__)
__A						:							int							=  {"""vocab_file""": """spiece.model"""}

__A						:							Optional[int]							=  {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

__A						:							Dict							=  {
    """albert-base-v1""": 5_1_2,
    """albert-large-v1""": 5_1_2,
    """albert-xlarge-v1""": 5_1_2,
    """albert-xxlarge-v1""": 5_1_2,
    """albert-base-v2""": 5_1_2,
    """albert-large-v2""": 5_1_2,
    """albert-xlarge-v2""": 5_1_2,
    """albert-xxlarge-v2""": 5_1_2,
}

__A						:							str							=  """▁"""




class 			UpperCAmelCase_      (   A       ):




       '''simple docstring'''



       a__						  =     VOCAB_FILES_NAMES
       a__						  =     PRETRAINED_VOCAB_FILES_MAP
       a__						  =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





       def __init__(					self       :	List[Any]       ,     a       :	Optional[int]       ,     a       :	Dict=True       ,     a       :	Tuple=True       ,     a       :	List[Any]=False       ,     a       :	Dict="[CLS]"       ,     a       :	List[Any]="[SEP]"       ,     a       :	Optional[int]="<unk>"       ,     a       :	List[str]="[SEP]"       ,     a       :	Union[str, Any]="<pad>"       ,     a       :	Optional[int]="[CLS]"       ,     a       :	Any="[MASK]"       ,     a       :	Optional[Dict[str, Any]] = None       ,     **a       :	Tuple       ,     )			->					None:
          # Mask token behave like a normal word, i.e. include the space before it and
          # is included in the raw text, there should be a match in a non-normalized sentence.
          SCREAMING_SNAKE_CASE           =					(
              AddedToken(a       ,     lstrip=a       ,     rstrip=a       ,     normalized=a			)
              if isinstance(a       ,     a			)
              else mask_token
          )

          SCREAMING_SNAKE_CASE           =					{} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              do_lower_case=a       ,     remove_space=a       ,     keep_accents=a       ,     bos_token=a       ,     eos_token=a       ,     unk_token=a       ,     sep_token=a       ,     pad_token=a       ,     cls_token=a       ,     mask_token=a       ,     sp_model_kwargs=self.sp_model_kwargs       ,     **a       ,     )

          SCREAMING_SNAKE_CASE           =					do_lower_case
          SCREAMING_SNAKE_CASE           =					remove_space
          SCREAMING_SNAKE_CASE           =					keep_accents
          SCREAMING_SNAKE_CASE           =					vocab_file

          SCREAMING_SNAKE_CASE           =					spm.SentencePieceProcessor(**self.sp_model_kwargs			)
          self.sp_model.Load(a			)





       @property
       def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					int:
          return len(self.sp_model			)





       def    _UpperCAmelCase     (					self       :	Any			)			->					Optional[Any]:
          SCREAMING_SNAKE_CASE           =					{self.convert_ids_to_tokens(a			): i for i in range(self.vocab_size			)}
          vocab.update(self.added_tokens_encoder			)
          return vocab





       def __getstate__(					self       :	Union[str, Any]			)			->					List[str]:
          SCREAMING_SNAKE_CASE           =					self.__dict__.copy()
          SCREAMING_SNAKE_CASE           =					None
          return state





       def __setstate__(					self       :	Union[str, Any]       ,     a       :	str			)			->					Optional[Any]:
          SCREAMING_SNAKE_CASE           =					d

          # for backward compatibility
          if not hasattr(self       ,     """sp_model_kwargs"""			):
             SCREAMING_SNAKE_CASE           =					{}

          SCREAMING_SNAKE_CASE           =					spm.SentencePieceProcessor(**self.sp_model_kwargs			)
          self.sp_model.Load(self.vocab_file			)





       def    _UpperCAmelCase     (					self       :	Dict       ,     a       :	Union[str, Any]			)			->					Optional[int]:
          if self.remove_space:
             SCREAMING_SNAKE_CASE           =					""" """.join(inputs.strip().split()			)
          else:
             SCREAMING_SNAKE_CASE           =					inputs
          SCREAMING_SNAKE_CASE           =					outputs.replace("""``"""       ,     """\""""			).replace("""''"""       ,     """\""""			)

          if not self.keep_accents:
             SCREAMING_SNAKE_CASE           =					unicodedata.normalize("""NFKD"""       ,     a			)
             SCREAMING_SNAKE_CASE           =					"""""".join([c for c in outputs if not unicodedata.combining(a			)]			)
          if self.do_lower_case:
             SCREAMING_SNAKE_CASE           =					outputs.lower()

          return outputs





       def    _UpperCAmelCase     (					self       :	List[str]       ,     a       :	str			)			->					List[str]:
          SCREAMING_SNAKE_CASE           =					self.preprocess_text(a			)
          SCREAMING_SNAKE_CASE           =					self.sp_model.encode(a       ,     out_type=a			)
          SCREAMING_SNAKE_CASE           =					[]
          for piece in pieces:
             if len(a			) > 1 and piece[-1] == str(""","""			) and piece[-2].isdigit():
                SCREAMING_SNAKE_CASE           =					self.sp_model.EncodeAsPieces(piece[:-1].replace(a       ,     """"""			)			)
                if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                   if len(cur_pieces[0]			) == 1:
                      SCREAMING_SNAKE_CASE           =					cur_pieces[1:]
                   else:
                      SCREAMING_SNAKE_CASE           =					cur_pieces[0][1:]
                cur_pieces.append(piece[-1]			)
                new_pieces.extend(a			)
             else:
                new_pieces.append(a			)

          return new_pieces





       def    _UpperCAmelCase     (					self       :	Any       ,     a       :	int			)			->					List[Any]:
          return self.sp_model.PieceToId(a			)





       def    _UpperCAmelCase     (					self       :	Tuple       ,     a       :	List[Any]			)			->					Any:
          return self.sp_model.IdToPiece(a			)





       def    _UpperCAmelCase     (					self       :	Tuple       ,     a       :	List[str]			)			->					Union[str, Any]:
          SCREAMING_SNAKE_CASE           =					[]
          SCREAMING_SNAKE_CASE           =					""""""
          SCREAMING_SNAKE_CASE           =					False
          for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                if not prev_is_special:
                   out_string += " "
                out_string += self.sp_model.decode(a			) + token
                SCREAMING_SNAKE_CASE           =					True
                SCREAMING_SNAKE_CASE           =					[]
             else:
                current_sub_tokens.append(a			)
                SCREAMING_SNAKE_CASE           =					False
          out_string += self.sp_model.decode(a			)
          return out_string.strip()





       def    _UpperCAmelCase     (					self       :	Optional[Any]       ,     a       :	List[int]       ,     a       :	Optional[List[int]] = None			)			->					List[int]:
          SCREAMING_SNAKE_CASE           =					[self.sep_token_id]
          SCREAMING_SNAKE_CASE           =					[self.cls_token_id]
          if token_ids_a is None:
             return cls + token_ids_a + sep
          return cls + token_ids_a + sep + token_ids_a + sep





       def    _UpperCAmelCase     (					self       :	int       ,     a       :	List[int]       ,     a       :	Optional[List[int]] = None       ,     a       :	bool = False			)			->					List[int]:

          if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=a       ,     token_ids_a=a       ,     already_has_special_tokens=a			)

          if token_ids_a is not None:
             return [1] + ([0] * len(a			)) + [1] + ([0] * len(a			)) + [1]
          return [1] + ([0] * len(a			)) + [1]





       def    _UpperCAmelCase     (					self       :	Any       ,     a       :	List[int]       ,     a       :	Optional[List[int]] = None			)			->					List[int]:
          SCREAMING_SNAKE_CASE           =					[self.sep_token_id]
          SCREAMING_SNAKE_CASE           =					[self.cls_token_id]

          if token_ids_a is None:
             return len(cls + token_ids_a + sep			) * [0]
          return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







       def    _UpperCAmelCase     (					self       :	List[Any]       ,     a       :	str       ,     a       :	Optional[str] = None			)			->					Tuple[str]:
          if not os.path.isdir(a			):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""			)
             return
          SCREAMING_SNAKE_CASE           =					os.path.join(
              a       ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

          if os.path.abspath(self.vocab_file			) != os.path.abspath(a			) and os.path.isfile(self.vocab_file			):
             copyfile(self.vocab_file       ,     a			)
          elif not os.path.isfile(self.vocab_file			):
             with open(a       ,     """wb"""			) as fi:
                SCREAMING_SNAKE_CASE           =					self.sp_model.serialized_model_proto()
                fi.write(a			)

          return (out_vocab_file,)












import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     ['''image_processor''', '''tokenizer''']
							a__						  =     '''ViTImageProcessor'''
							a__						  =     ('''CLIPTokenizer''', '''CLIPTokenizerFast''')





							def __init__(					self       :	List[str]       ,     a       :	Dict=None       ,     a       :	List[str]=None       ,     **a       :	List[str]			)			->					Optional[Any]:
										SCREAMING_SNAKE_CASE           =					None
										if "feature_extractor" in kwargs:
													warnings.warn(
													    """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
													    """ instead."""       ,     a       ,     )
													SCREAMING_SNAKE_CASE           =					kwargs.pop("""feature_extractor"""			)

										SCREAMING_SNAKE_CASE           =					image_processor if image_processor is not None else feature_extractor
										if image_processor is None:
													raise ValueError("""You need to specify an `image_processor`."""			)
										if tokenizer is None:
													raise ValueError("""You need to specify a `tokenizer`."""			)

										super().__init__(a       ,     a			)





							def __call__(					self       :	List[Any]       ,     a       :	int=None       ,     a       :	Dict=None       ,     a       :	int=None       ,     a       :	Tuple=None       ,     **a       :	Any			)			->					Any:
										if text is None and visual_prompt is None and images is None:
													raise ValueError("""You have to specify either text, visual prompt or images."""			)

										if text is not None and visual_prompt is not None:
													raise ValueError("""You have to specify exactly one type of prompt. Either text or visual prompt."""			)

										if text is not None:
													SCREAMING_SNAKE_CASE           =					self.tokenizer(a       ,     return_tensors=a       ,     **a			)

										if visual_prompt is not None:
													SCREAMING_SNAKE_CASE           =					self.image_processor(a       ,     return_tensors=a       ,     **a			)

										if images is not None:
													SCREAMING_SNAKE_CASE           =					self.image_processor(a       ,     return_tensors=a       ,     **a			)

										if visual_prompt is not None and images is not None:
													SCREAMING_SNAKE_CASE           =					{
													    """pixel_values""": image_features.pixel_values,
													    """conditional_pixel_values""": prompt_features.pixel_values,
													}
													return encoding
										elif text is not None and images is not None:
													SCREAMING_SNAKE_CASE           =					image_features.pixel_values
													return encoding
										elif text is not None:
													return encoding
										elif visual_prompt is not None:
													SCREAMING_SNAKE_CASE           =					{
													    """conditional_pixel_values""": prompt_features.pixel_values,
													}
													return encoding
										else:
													return BatchEncoding(data=dict(**a			)       ,     tensor_type=a			)





							def    _UpperCAmelCase     (					self       :	Dict       ,     *a       :	Dict       ,     **a       :	Union[str, Any]			)			->					Any:
										return self.tokenizer.batch_decode(*a       ,     **a			)





							def    _UpperCAmelCase     (					self       :	Tuple       ,     *a       :	List[Any]       ,     **a       :	Union[str, Any]			)			->					Dict:
										return self.tokenizer.decode(*a       ,     **a			)





							@property
							def    _UpperCAmelCase     (					self       :	Any			)			->					Optional[Any]:
										warnings.warn(
										    """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""       ,     a       ,     )
										return self.image_processor_class







							@property
							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					str:
										warnings.warn(
										    """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""       ,     a       ,     )
										return self.image_processor






