






"""simple docstring"""
import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_       			=   logging.get_logger(__name__)

lowerCamelCase_       			=   '''▁'''

lowerCamelCase_       			=   {
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
}

lowerCamelCase_       			=   {
    '''vocab_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'''
        ),
    },
    '''spm_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'''
        )
    },
}

lowerCamelCase_       			=   {
    '''facebook/s2t-small-librispeech-asr''': 1024,
}

lowerCamelCase_       			=   ['''pt''', '''fr''', '''ru''', '''nl''', '''ro''', '''it''', '''es''', '''de''']

lowerCamelCase_       			=   {'''mustc''': MUSTC_LANGS}







class     UpperCamelCase_ (__A  ):
	__magic_name__			   =       VOCAB_FILES_NAMES
	__magic_name__			   =       PRETRAINED_VOCAB_FILES_MAP
	__magic_name__			   =       MAX_MODEL_INPUT_SIZES
	__magic_name__			   =       ['''input_ids''', '''attention_mask''']

	__magic_name__			   =       []





	def __init__(   self      :				List[str]			,			lowerCAmelCase_      :				Optional[Any]			,			lowerCAmelCase_      :				List[str]			,			lowerCAmelCase_      :				List[str]="<s>"			,			lowerCAmelCase_      :				Union[str, Any]="</s>"			,			lowerCAmelCase_      :				Tuple="<pad>"			,			lowerCAmelCase_      :				Any="<unk>"			,			lowerCAmelCase_      :				Optional[int]=False			,			lowerCAmelCase_      :				Any=False			,			lowerCAmelCase_      :				Dict=None			,			lowerCAmelCase_      :				Tuple=None			,			lowerCAmelCase_      :				Optional[Dict[str, Any]] = None			,			**lowerCAmelCase_      :				Any			,			)		->    None:
				UpperCAmelCase_  :				int      =      {} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    bos_token=lowerCAmelCase_			,			eos_token=lowerCAmelCase_			,			unk_token=lowerCAmelCase_			,			pad_token=lowerCAmelCase_			,			do_upper_case=lowerCAmelCase_			,			do_lower_case=lowerCAmelCase_			,			tgt_lang=lowerCAmelCase_			,			lang_codes=lowerCAmelCase_			,			sp_model_kwargs=self.sp_model_kwargs			,			**lowerCAmelCase_			,			)
				UpperCAmelCase_  :				Any      =      do_upper_case
				UpperCAmelCase_  :				str      =      do_lower_case

				UpperCAmelCase_  :				Optional[Any]      =      load_json(lowerCAmelCase_		)
				UpperCAmelCase_  :				List[Any]      =      {v: k for k, v in self.encoder.items()}
				UpperCAmelCase_  :				List[str]      =      spm_file
				UpperCAmelCase_  :				str      =      load_spm(lowerCAmelCase_			,			self.sp_model_kwargs		)

				if lang_codes is not None:
							UpperCAmelCase_  :				List[str]      =      lang_codes
							UpperCAmelCase_  :				Tuple      =      LANGUAGES[lang_codes]
							UpperCAmelCase_  :				str      =      [f"""<lang:{lang}>""" for lang in self.langs]
							UpperCAmelCase_  :				int      =      {lang: self.sp_model.PieceToId(f"""<lang:{lang}>"""		) for lang in self.langs}

							UpperCAmelCase_  :				int      =      self.lang_tokens
							UpperCAmelCase_  :				List[str]      =      tgt_lang if tgt_lang is not None else self.langs[0]

							self.set_tgt_lang_special_tokens(self._tgt_lang		)
				else:
							UpperCAmelCase_  :				int      =      {}





	@property
	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[int]		)		->    int:
				return len(self.encoder		)





	@property
	def   _SCREAMING_SNAKE_CASE   (   self      :				List[Any]		)		->    str:
				return self._tgt_lang





	@tgt_lang.setter
	def   _SCREAMING_SNAKE_CASE   (   self      :				Any			,			lowerCAmelCase_      :				Any		)		->    None:
				UpperCAmelCase_  :				str      =      new_tgt_lang
				self.set_tgt_lang_special_tokens(lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[Any]			,			lowerCAmelCase_      :				str		)		->    None:
				UpperCAmelCase_  :				Tuple      =      self.lang_code_to_id[tgt_lang]
				UpperCAmelCase_  :				Tuple      =      [lang_code_id]





	def   _SCREAMING_SNAKE_CASE   (   self      :				Any			,			lowerCAmelCase_      :				str		)		->    List[str]:
				return self.sp_model.encode(lowerCAmelCase_			,			out_type=lowerCAmelCase_		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Optional[int]			,			lowerCAmelCase_      :				Dict		)		->    Union[str, Any]:
				return self.encoder.get(lowerCAmelCase_			,			self.encoder[self.unk_token]		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Union[str, Any]			,			lowerCAmelCase_      :				int		)		->    str:
				return self.decoder.get(lowerCAmelCase_			,			self.unk_token		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				Any			,			lowerCAmelCase_      :				List[str]		)		->    str:
				UpperCAmelCase_  :				str      =      []
				UpperCAmelCase_  :				Any      =      ""
				for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
										UpperCAmelCase_  :				List[Any]      =      self.sp_model.decode(lowerCAmelCase_		)
										out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
										UpperCAmelCase_  :				List[Any]      =      []
							else:
										current_sub_tokens.append(lowerCAmelCase_		)
				UpperCAmelCase_  :				List[Any]      =      self.sp_model.decode(lowerCAmelCase_		)
				out_string += decoded.upper() if self.do_upper_case else decoded
				return out_string.strip()





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[str]			,			lowerCAmelCase_      :				Any			,			lowerCAmelCase_      :				Optional[Any]=None		)		->    List[int]:
				if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + [self.eos_token_id]
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]





	def   _SCREAMING_SNAKE_CASE   (   self      :				int			,			lowerCAmelCase_      :				List[int]			,			lowerCAmelCase_      :				Optional[List[int]] = None			,			lowerCAmelCase_      :				bool = False		)		->    List[int]:

				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowerCAmelCase_			,			token_ids_a=lowerCAmelCase_			,			already_has_special_tokens=lowerCAmelCase_		)

				UpperCAmelCase_  :				List[Any]      =      [1] * len(self.prefix_tokens		)
				UpperCAmelCase_  :				Optional[int]      =      [1]
				if token_ids_a is None:
							return prefix_ones + ([0] * len(lowerCAmelCase_		)) + suffix_ones
				return prefix_ones + ([0] * len(lowerCAmelCase_		)) + ([0] * len(lowerCAmelCase_		)) + suffix_ones





	def   _SCREAMING_SNAKE_CASE   (   self      :				str		)		->    Dict:
				UpperCAmelCase_  :				Any      =      self.encoder.copy()
				vocab.update(self.added_tokens_encoder		)
				return vocab





	def __getstate__(   self      :				Dict		)		->    Dict:
				UpperCAmelCase_  :				Optional[Any]      =      self.__dict__.copy()
				UpperCAmelCase_  :				Optional[int]      =      None
				return state





	def __setstate__(   self      :				Any			,			lowerCAmelCase_      :				Dict		)		->    None:
				UpperCAmelCase_  :				Any      =      d

				# for backward compatibility
				if not hasattr(self			,			"sp_model_kwargs"		):
							UpperCAmelCase_  :				List[Any]      =      {}

				UpperCAmelCase_  :				List[Any]      =      load_spm(self.spm_file			,			self.sp_model_kwargs		)





	def   _SCREAMING_SNAKE_CASE   (   self      :				List[Any]			,			lowerCAmelCase_      :				str			,			lowerCAmelCase_      :				Optional[str] = None		)		->    Tuple[str]:
				UpperCAmelCase_  :				str      =      Path(lowerCAmelCase_		)
				assert save_dir.is_dir(), f"""{save_directory} should be a directory"""
				UpperCAmelCase_  :				Union[str, Any]      =      save_dir / (
				    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
				)
				UpperCAmelCase_  :				List[str]      =      save_dir / (
				    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
				)

				save_json(self.encoder			,			lowerCAmelCase_		)

				if os.path.abspath(self.spm_file		) != os.path.abspath(lowerCAmelCase_		) and os.path.isfile(self.spm_file		):
							copyfile(self.spm_file			,			lowerCAmelCase_		)
				elif not os.path.isfile(self.spm_file		):
							with open(lowerCAmelCase_			,			"wb"		) as fi:
										UpperCAmelCase_  :				List[Any]      =      self.sp_model.serialized_model_proto()
										fi.write(lowerCAmelCase_		)

				return (str(lowerCAmelCase_		), str(lowerCAmelCase_		))





def        snake_case   (     A__			,A__		):

			'''simple docstring'''





			UpperCAmelCase_  :				Optional[Any]      =      sentencepiece.SentencePieceProcessor(**A__		)
			spm.Load(str(A__		)		)
			return spm

def        snake_case   (     A__		):

			'''simple docstring'''





			with open(A__			,"r"		) as f:
						return json.load(A__		)




def        snake_case   (     A__			,A__		):

			'''simple docstring'''





			with open(A__			,"w"		) as f:
						json.dump(A__			,A__			,indent=2		)











"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


lowerCamelCase_       			=   {
    '''configuration_clip''': [
        '''CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''CLIPConfig''',
        '''CLIPOnnxConfig''',
        '''CLIPTextConfig''',
        '''CLIPVisionConfig''',
    ],
    '''processing_clip''': ['''CLIPProcessor'''],
    '''tokenization_clip''': ['''CLIPTokenizer'''],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCamelCase_       			=   ['''CLIPTokenizerFast''']

try:
			if not is_vision_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCamelCase_       			=   ['''CLIPFeatureExtractor''']
			lowerCamelCase_       			=   ['''CLIPImageProcessor''']

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCamelCase_       			=   [
			    '''CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''CLIPModel''',
			    '''CLIPPreTrainedModel''',
			    '''CLIPTextModel''',
			    '''CLIPTextModelWithProjection''',
			    '''CLIPVisionModel''',
			    '''CLIPVisionModelWithProjection''',
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCamelCase_       			=   [
			    '''TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''TFCLIPModel''',
			    '''TFCLIPPreTrainedModel''',
			    '''TFCLIPTextModel''',
			    '''TFCLIPVisionModel''',
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowerCamelCase_       			=   [
			    '''FlaxCLIPModel''',
			    '''FlaxCLIPPreTrainedModel''',
			    '''FlaxCLIPTextModel''',
			    '''FlaxCLIPTextPreTrainedModel''',
			    '''FlaxCLIPVisionModel''',
			    '''FlaxCLIPVisionPreTrainedModel''',
			]


if TYPE_CHECKING:
			from .configuration_clip import (
			    CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
			    CLIPConfig,
			    CLIPOnnxConfig,
			    CLIPTextConfig,
			    CLIPVisionConfig,
			)
			from .processing_clip import CLIPProcessor
			from .tokenization_clip import CLIPTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_clip_fast import CLIPTokenizerFast

			try:
						if not is_vision_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .feature_extraction_clip import CLIPFeatureExtractor
						from .image_processing_clip import CLIPImageProcessor

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_clip import (
						    CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
						    CLIPModel,
						    CLIPPreTrainedModel,
						    CLIPTextModel,
						    CLIPTextModelWithProjection,
						    CLIPVisionModel,
						    CLIPVisionModelWithProjection,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_clip import (
						    TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFCLIPModel,
						    TFCLIPPreTrainedModel,
						    TFCLIPTextModel,
						    TFCLIPVisionModel,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_flax_clip import (
						    FlaxCLIPModel,
						    FlaxCLIPPreTrainedModel,
						    FlaxCLIPTextModel,
						    FlaxCLIPTextPreTrainedModel,
						    FlaxCLIPVisionModel,
						    FlaxCLIPVisionPreTrainedModel,
						)


else:
			import sys

			lowerCamelCase_       			=   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



