


"""simple docstring"""

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__  							=logging.get_logger(__name__)

UpperCAmelCase__  							="▁"

UpperCAmelCase__  							={"vocab_file": "sentencepiece.bpe.model"}

UpperCAmelCase__  							={
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

UpperCAmelCase__  							={
    "facebook/mbart-large-50-one-to-many-mmt": 1024,
}

# fmt: off
UpperCAmelCase__  							=["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]





class    lowerCamelCase__						(		_a    ):
      a :     int											=		VOCAB_FILES_NAMES
      a :     List[str]											=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a :     Optional[int]											=		PRETRAINED_VOCAB_FILES_MAP
      a :     int											=		["""input_ids""", """attention_mask"""]

      a :     List[int]											=		[]
      a :     List[int]											=		[]
      def __init__(		self	:	Tuple      ,       A_	:	str      ,       A_	:	Optional[Any]=None      ,       A_	:	int=None      ,       A_	:	Tuple="</s>"      ,       A_	:	Optional[Any]="</s>"      ,       A_	:	Union[str, Any]="<s>"      ,       A_	:	Optional[Any]="<unk>"      ,       A_	:	Optional[int]="<pad>"      ,       A_	:	List[Any]="<mask>"      ,       A_	:	Optional[Dict[str, Any]] = None      ,       **A_	:	Dict      ,       ):






       '''simple docstring'''


       __lowercase									=	AddedToken(A_      ,       lstrip=A_      ,       rstrip=A_							) if isinstance(A_      ,       A_							) else mask_token

       __lowercase									=	{} if sp_model_kwargs is None else sp_model_kwargs

       __lowercase									=	kwargs.get("""additional_special_tokens"""      ,       []							)
       kwargs["additional_special_tokens"] += [
           code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
       ]

       super().__init__(
           src_lang=A_      ,       tgt_lang=A_      ,       eos_token=A_      ,       unk_token=A_      ,       sep_token=A_      ,       cls_token=A_      ,       pad_token=A_      ,       mask_token=A_      ,       sp_model_kwargs=self.sp_model_kwargs      ,       **A_      ,       )

       __lowercase									=	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
       self.sp_model.Load(str(A_							)							)
       __lowercase									=	vocab_file

       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
       # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
       # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

       # Mimic fairseq token-to-id alignment for the first 4 token
       __lowercase									=	{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       __lowercase									=	1

       __lowercase									=	len(self.sp_model							)
       __lowercase									=	{
           code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(A_							)
       }
       __lowercase									=	{v: k for k, v in self.lang_code_to_id.items()}
       __lowercase									=	len(self.sp_model							) + len(self.lang_code_to_id							) + self.fairseq_offset

       self.fairseq_tokens_to_ids.update(self.lang_code_to_id							)
       __lowercase									=	{v: k for k, v in self.fairseq_tokens_to_ids.items()}

       __lowercase									=	src_lang if src_lang is not None else """en_XX"""
       __lowercase									=	self.lang_code_to_id[self._src_lang]
       __lowercase									=	tgt_lang
       self.set_src_lang_special_tokens(self._src_lang							)
      @property
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]							):






       '''simple docstring'''


       return len(self.sp_model							) + len(self.lang_code_to_id							) + self.fairseq_offset + 1  # Plus 1 for the mask token
      @property
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       return self._src_lang
      @src_lang.setter
      def SCREAMING_SNAKE_CASE_	(		self	:	Any      ,       A_	:	str							):






       '''simple docstring'''


       __lowercase									=	new_src_lang
       self.set_src_lang_special_tokens(self._src_lang							)
      def __getstate__(		self	:	str							):






       '''simple docstring'''


       __lowercase									=	self.__dict__.copy()
       __lowercase									=	None
       return state
      def __setstate__(		self	:	int      ,       A_	:	Dict							):






       '''simple docstring'''


       __lowercase									=	d

       # for backward compatibility
       if not hasattr(self      ,       """sp_model_kwargs"""							):
        __lowercase									=	{}

       __lowercase									=	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
       self.sp_model.Load(self.vocab_file							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[int]							):






       '''simple docstring'''


       __lowercase									=	{self.convert_ids_to_tokens(A_							): i for i in range(self.vocab_size							)}
       vocab.update(self.added_tokens_encoder							)
       return vocab
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[int]      ,       A_	:	str							):






       '''simple docstring'''


       return self.sp_model.encode(A_      ,       out_type=A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]      ,       A_	:	str							):






       '''simple docstring'''


       if token in self.fairseq_tokens_to_ids:
        return self.fairseq_tokens_to_ids[token]
       __lowercase									=	self.sp_model.PieceToId(A_							)

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple      ,       A_	:	int							):






       '''simple docstring'''


       if index in self.fairseq_ids_to_tokens:
        return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset							)
      def SCREAMING_SNAKE_CASE_	(		self	:	int      ,       A_	:	Tuple							):






       '''simple docstring'''


       __lowercase									=	[]
       __lowercase									=	""""""
       __lowercase									=	False
       for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
         if not prev_is_special:
          out_string += " "
         out_string += self.sp_model.decode(A_							) + token
         __lowercase									=	True
         __lowercase									=	[]
        else:
         current_sub_tokens.append(A_							)
         __lowercase									=	False
       out_string += self.sp_model.decode(A_							)
       return out_string.strip()
      def SCREAMING_SNAKE_CASE_	(		self	:	List[str]      ,       A_	:	str      ,       A_	:	Optional[str] = None							):






       '''simple docstring'''


       if not os.path.isdir(A_							):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
        return
       __lowercase									=	os.path.join(
           A_      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

       if os.path.abspath(self.vocab_file							) != os.path.abspath(A_							) and os.path.isfile(self.vocab_file							):
        copyfile(self.vocab_file      ,       A_							)
       elif not os.path.isfile(self.vocab_file							):
        with open(A_      ,       """wb"""							) as fi:
         __lowercase									=	self.sp_model.serialized_model_proto()
         fi.write(A_							)

       return (out_vocab_file,)
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]      ,       A_	:	List[int]      ,       A_	:	Optional[List[int]] = None      ,       A_	:	bool = False							):






       '''simple docstring'''


       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=A_      ,       token_ids_a=A_      ,       already_has_special_tokens=A_							)

       __lowercase									=	[1] * len(self.prefix_tokens							)
       __lowercase									=	[1] * len(self.suffix_tokens							)
       if token_ids_a is None:
        return prefix_ones + ([0] * len(A_							)) + suffix_ones
       return prefix_ones + ([0] * len(A_							)) + ([0] * len(A_							)) + suffix_ones
      def SCREAMING_SNAKE_CASE_	(		self	:	str      ,       A_	:	List[int]      ,       A_	:	Optional[List[int]] = None							):






       '''simple docstring'''


       if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
      def SCREAMING_SNAKE_CASE_	(		self	:	int      ,       A_	:	List[str]      ,       A_	:	str      ,       A_	:	Optional[str]      ,       A_	:	Optional[str]      ,       **A_	:	Dict							):






       '''simple docstring'''


       if src_lang is None or tgt_lang is None:
        raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""							)
       __lowercase									=	src_lang
       __lowercase									=	self(A_      ,       add_special_tokens=A_      ,       return_tensors=A_      ,       **A_							)
       __lowercase									=	self.convert_tokens_to_ids(A_							)
       __lowercase									=	tgt_lang_id
       return inputs
      def SCREAMING_SNAKE_CASE_	(		self	:	List[Any]      ,       A_	:	List[str]      ,       A_	:	str = "en_XX"      ,       A_	:	Optional[List[str]] = None      ,       A_	:	str = "ro_RO"      ,       **A_	:	Optional[int]      ,       ):






       '''simple docstring'''


       __lowercase									=	src_lang
       __lowercase									=	tgt_lang
       return super().prepare_seqaseq_batch(A_      ,       A_      ,       **A_							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Any							):






       '''simple docstring'''


       return self.set_src_lang_special_tokens(self.src_lang							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple							):






       '''simple docstring'''


       return self.set_tgt_lang_special_tokens(self.tgt_lang							)
      def SCREAMING_SNAKE_CASE_	(		self	:	Optional[Any]      ,       A_	:	str							):






       '''simple docstring'''


       __lowercase									=	self.lang_code_to_id[src_lang]
       __lowercase									=	[self.cur_lang_code_id]
       __lowercase									=	[self.eos_token_id]






      def SCREAMING_SNAKE_CASE_	(		self	:	Tuple      ,       A_	:	str							):






       '''simple docstring'''


       __lowercase									=	self.lang_code_to_id[tgt_lang]
       __lowercase									=	[self.cur_lang_code_id]
       __lowercase									=	[self.eos_token_id]




"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__  							=logging.get_logger(__name__)

UpperCAmelCase__  							={
    "microsoft/swinv2-tiny-patch4-window8-256": (
        "https://huggingface.co/microsoft/swinv2-tiny-patch4-window8-256/resolve/main/config.json"
    ),
}





class    lowerCamelCase__						(		_a    ):
      a :     List[Any]											=		"""swinv2"""

      a :     Any											=		{
          """num_attention_heads""": """num_heads""",
          """num_hidden_layers""": """num_layers""",
      }
      def __init__(		self	:	List[str]      ,       A_	:	Any=2_2_4      ,       A_	:	int=4      ,       A_	:	Optional[int]=3      ,       A_	:	List[Any]=9_6      ,       A_	:	List[Any]=[2, 2, 6, 2]      ,       A_	:	List[str]=[3, 6, 1_2, 2_4]      ,       A_	:	Union[str, Any]=7      ,       A_	:	int=4.0      ,       A_	:	List[str]=True      ,       A_	:	str=0.0      ,       A_	:	Any=0.0      ,       A_	:	Union[str, Any]=0.1      ,       A_	:	Optional[Any]="gelu"      ,       A_	:	int=False      ,       A_	:	List[Any]=0.02      ,       A_	:	Tuple=1e-5      ,       A_	:	Tuple=3_2      ,       **A_	:	int      ,       ):






       '''simple docstring'''


       super().__init__(**A_							)

       __lowercase									=	image_size
       __lowercase									=	patch_size
       __lowercase									=	num_channels
       __lowercase									=	embed_dim
       __lowercase									=	depths
       __lowercase									=	len(A_							)
       __lowercase									=	num_heads
       __lowercase									=	window_size
       __lowercase									=	mlp_ratio
       __lowercase									=	qkv_bias
       __lowercase									=	hidden_dropout_prob
       __lowercase									=	attention_probs_dropout_prob
       __lowercase									=	drop_path_rate
       __lowercase									=	hidden_act
       __lowercase									=	use_absolute_embeddings
       __lowercase									=	layer_norm_eps
       __lowercase									=	initializer_range
       __lowercase									=	encoder_stride
       # we set the hidden_size attribute in order to make Swinv2 work with VisionEncoderDecoderModel
       # this indicates the channel dimension after the last stage of the model
       __lowercase									=	int(embed_dim * 2 ** (len(A_							) - 1)							)
       __lowercase									=	(0, 0, 0, 0)
