




"""simple docstring"""


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


A_      			=       logging.get_logger(__name__)

A_      			=       {'''vocab_file''': '''spiece.model'''}

A_      			=       {
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

A_      			=       {
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
A_      			=       0
A_      			=       1
A_      			=       2
A_      			=       3
A_      			=       4






class       lowercase(	__lowerCamelCase				):







 '''simple docstring'''





 lowercase__				      =      VOCAB_FILES_NAMES
 lowercase__				      =      PRETRAINED_VOCAB_FILES_MAP
 lowercase__				      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase__				      =      '''left'''


 def __init__(       self:	int,   a_:	List[Any],   a_:	int=False,   a_:	Tuple=True,   a_:	Dict=False,   a_:	Any="<s>",   a_:	int="</s>",   a_:	List[str]="<unk>",   a_:	str="<sep>",   a_:	Tuple="<pad>",   a_:	int="<cls>",   a_:	Any="<mask>",   a_:	Tuple=["<eop>", "<eod>"],   a_:	List[str] = None,   **a_:	int,   ):




   '''simple docstring'''



   _snake_case							:	str			=						AddedToken(a_,   lstrip=a_,   rstrip=a_	) if isinstance(a_,   a_	) else mask_token

   _snake_case							:	int			=						{} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=a_,   remove_space=a_,   keep_accents=a_,   bos_token=a_,   eos_token=a_,   unk_token=a_,   sep_token=a_,   pad_token=a_,   cls_token=a_,   mask_token=a_,   additional_special_tokens=a_,   sp_model_kwargs=self.sp_model_kwargs,   **a_,   )

   _snake_case							:	str			=						3

   _snake_case							:	Dict			=						do_lower_case
   _snake_case							:	str			=						remove_space
   _snake_case							:	Tuple			=						keep_accents
   _snake_case							:	Dict			=						vocab_file

   _snake_case							:	List[str]			=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
   self.sp_model.Load(a_	)


 @property
 def 				UpperCamelCase_     (       self:	int	):




   '''simple docstring'''



   return len(self.sp_model	)


 def 				UpperCamelCase_     (       self:	Optional[Any]	):




   '''simple docstring'''



   _snake_case							:	List[str]			=						{self.convert_ids_to_tokens(a_	): i for i in range(self.vocab_size	)}
   vocab.update(self.added_tokens_encoder	)
   return vocab


 def __getstate__(       self:	Any	):




   '''simple docstring'''



   _snake_case							:	Any			=						self.__dict__.copy()
   _snake_case							:	Optional[int]			=						None
   return state


 def __setstate__(       self:	Union[str, Any],   a_:	Dict	):




   '''simple docstring'''



   _snake_case							:	Union[str, Any]			=						d

   # for backward compatibility
   if not hasattr(self,   """sp_model_kwargs"""	):
     _snake_case							:	int			=						{}

   _snake_case							:	Union[str, Any]			=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
   self.sp_model.Load(self.vocab_file	)


 def 				UpperCamelCase_     (       self:	int,   a_:	Tuple	):




   '''simple docstring'''



   if self.remove_space:
     _snake_case							:	Optional[int]			=						" ".join(inputs.strip().split()	)
   else:
     _snake_case							:	str			=						inputs
   _snake_case							:	Any			=						outputs.replace("""``""",   """\""""	).replace("""''""",   """\""""	)

   if not self.keep_accents:
     _snake_case							:	Dict			=						unicodedata.normalize("""NFKD""",   a_	)
     _snake_case							:	int			=						"".join([c for c in outputs if not unicodedata.combining(a_	)]	)
   if self.do_lower_case:
     _snake_case							:	Any			=						outputs.lower()

   return outputs


 def 				UpperCamelCase_     (       self:	Dict,   a_:	Optional[int]	):




   '''simple docstring'''



   _snake_case							:	List[Any]			=						self.preprocess_text(a_	)
   _snake_case							:	Optional[int]			=						self.sp_model.encode(a_,   out_type=a_	)
   _snake_case							:	List[str]			=						[]
   for piece in pieces:
     if len(a_	) > 1 and piece[-1] == str(""","""	) and piece[-2].isdigit():
       _snake_case							:	Tuple			=						self.sp_model.EncodeAsPieces(piece[:-1].replace(a_,   """"""	)	)
       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
         if len(cur_pieces[0]	) == 1:
           _snake_case							:	int			=						cur_pieces[1:]
         else:
           _snake_case							:	Union[str, Any]			=						cur_pieces[0][1:]
       cur_pieces.append(piece[-1]	)
       new_pieces.extend(a_	)
     else:
       new_pieces.append(a_	)

   return new_pieces


 def 				UpperCamelCase_     (       self:	List[str],   a_:	Union[str, Any]	):




   '''simple docstring'''



   return self.sp_model.PieceToId(a_	)


 def 				UpperCamelCase_     (       self:	Optional[int],   a_:	Tuple	):




   '''simple docstring'''



   return self.sp_model.IdToPiece(a_	)


 def 				UpperCamelCase_     (       self:	str,   a_:	List[Any]	):




   '''simple docstring'''



   _snake_case							:	Dict			=						"".join(a_	).replace(a_,   """ """	).strip()
   return out_string


 def 				UpperCamelCase_     (       self:	Dict,   a_:	int,   a_:	Dict = False,   a_:	Optional[Any] = None,   a_:	List[Any] = True,   **a_:	Dict,   ):




   '''simple docstring'''



   _snake_case							:	int			=						kwargs.pop("""use_source_tokenizer""",   a_	)

   _snake_case							:	List[str]			=						self.convert_ids_to_tokens(a_,   skip_special_tokens=a_	)

   # To avoid mixing byte-level and unicode for byte-level BPT
   # we need to build string separately for added tokens and byte-level tokens
   # cf. https://github.com/huggingface/transformers/issues/1133
   _snake_case							:	Optional[int]			=						[]
   _snake_case							:	List[str]			=						[]
   for token in filtered_tokens:
     if skip_special_tokens and token in self.all_special_ids:
       continue
     if token in self.added_tokens_encoder:
       if current_sub_text:
         sub_texts.append(self.convert_tokens_to_string(a_	)	)
         _snake_case							:	Union[str, Any]			=						[]
       sub_texts.append(a_	)
     else:
       current_sub_text.append(a_	)
   if current_sub_text:
     sub_texts.append(self.convert_tokens_to_string(a_	)	)

   # Mimic the behavior of the Rust tokenizer:
   # By default, there are no spaces between special tokens
   _snake_case							:	Union[str, Any]			=						"".join(a_	)

   _snake_case							:	Optional[Any]			=						(
       clean_up_tokenization_spaces
       if clean_up_tokenization_spaces is not None
       else self.clean_up_tokenization_spaces
   )
   if clean_up_tokenization_spaces:
     _snake_case							:	List[Any]			=						self.clean_up_tokenization(a_	)
     return clean_text
   else:
     return text


 def 				UpperCamelCase_     (       self:	Any,   a_:	List[Any],   a_:	List[str] = None	):




   '''simple docstring'''



   _snake_case							:	Optional[Any]			=						[self.sep_token_id]
   _snake_case							:	Union[str, Any]			=						[self.cls_token_id]
   if token_ids_a is None:
     return token_ids_a + sep + cls
   return token_ids_a + sep + token_ids_a + sep + cls


 def 				UpperCamelCase_     (       self:	Union[str, Any],   a_:	str,   a_:	Tuple = None,   a_:	Optional[Any] = False	):




   '''simple docstring'''



   if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=a_,   token_ids_a=a_,   already_has_special_tokens=a_	)

   if token_ids_a is not None:
     return ([0] * len(a_	)) + [1] + ([0] * len(a_	)) + [1, 1]
   return ([0] * len(a_	)) + [1, 1]


 def 				UpperCamelCase_     (       self:	Union[str, Any],   a_:	Any,   a_:	Optional[Any] = None	):




   '''simple docstring'''



   _snake_case							:	Optional[Any]			=						[self.sep_token_id]
   _snake_case							:	Union[str, Any]			=						[2]

   if token_ids_a is None:
     return len(token_ids_a + sep	) * [0] + cls_segment_id
   return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id


 def 				UpperCamelCase_     (       self:	List[Any],   a_:	str,   a_:	Optional[Any] = None	):




   '''simple docstring'''



   if not os.path.isdir(a_	):
     logger.error(f"Vocabulary path ({save_directory}) should be a directory"	)
     return
   _snake_case							:	Any			=						os.path.join(
       a_,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

   if os.path.abspath(self.vocab_file	) != os.path.abspath(a_	) and os.path.isfile(self.vocab_file	):
     copyfile(self.vocab_file,   a_	)
   elif not os.path.isfile(self.vocab_file	):
     with open(a_,   """wb"""	) as fi:
       _snake_case							:	Dict			=						self.sp_model.serialized_model_proto()
       fi.write(a_	)

   return (out_vocab_file,)








"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


A_      			=       {
    '''configuration_owlvit''': [
        '''OWLVIT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''OwlViTConfig''',
        '''OwlViTOnnxConfig''',
        '''OwlViTTextConfig''',
        '''OwlViTVisionConfig''',
    ],
    '''processing_owlvit''': ['''OwlViTProcessor'''],
}


try:
     if not is_vision_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     A_      			=       ['''OwlViTFeatureExtractor''']
     A_      			=       ['''OwlViTImageProcessor''']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     A_      			=       [
         '''OWLVIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''OwlViTModel''',
         '''OwlViTPreTrainedModel''',
         '''OwlViTTextModel''',
         '''OwlViTVisionModel''',
         '''OwlViTForObjectDetection''',
     ]

if TYPE_CHECKING:
     from .configuration_owlvit import (
         OWLVIT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         OwlViTConfig,
         OwlViTOnnxConfig,
         OwlViTTextConfig,
         OwlViTVisionConfig,
     )
     from .processing_owlvit import OwlViTProcessor

     try:
          if not is_vision_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .feature_extraction_owlvit import OwlViTFeatureExtractor
          from .image_processing_owlvit import OwlViTImageProcessor

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_owlvit import (
              OWLVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
              OwlViTForObjectDetection,
              OwlViTModel,
              OwlViTPreTrainedModel,
              OwlViTTextModel,
              OwlViTVisionModel,
          )

else:
     import sys

     A_      			=       _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


